From d3187b16c0f4f840133ed7d65827d912a9672a51 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 May 2020 20:06:39 +0200 Subject: [PATCH 01/13] Removed outlook hook --- functions/triggers/outlook/README.md | 57 ---- functions/triggers/outlook/hook.go | 222 ------------- .../triggers/outlook/integrations/config.json | 7 - .../triggers/outlook/integrations/server.crt | 21 -- .../triggers/outlook/integrations/server.key | 27 -- functions/triggers/outlook/main.go | 41 --- functions/triggers/outlook/run.go | 304 ------------------ 7 files changed, 679 deletions(-) delete mode 100644 functions/triggers/outlook/README.md delete mode 100644 functions/triggers/outlook/hook.go delete mode 100644 functions/triggers/outlook/integrations/config.json delete mode 100644 functions/triggers/outlook/integrations/server.crt delete mode 100644 functions/triggers/outlook/integrations/server.key delete mode 100644 functions/triggers/outlook/main.go delete mode 100644 functions/triggers/outlook/run.go diff --git a/functions/triggers/outlook/README.md b/functions/triggers/outlook/README.md deleted file mode 100644 index c1c5db5d..00000000 --- a/functions/triggers/outlook/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Outlook trigger -Makes it possible to trigger a workflow based on an email - -## Local testing - Same as ../webhook -```bash -mv ../main.go -go run main.go hook.go -``` - -# Deploy gcloud -gcloud functions deploy outlooktrigger --runtime go111 --entry-point Authorization --trigger-http --project shuffler --memory=128 --set-env-vars=FUNCTION_APIKEY=asdasd,CALLBACKURL=shuffler.io,TRIGGERID=test123,WORKFLOW_ID=YOUR_WORKFLOW_ID - -# Build and deploy -1. Set hook.go line 1 from "package main" to "package function" -2. zip outlooktrigger.tar hook.go -3. Upload to bucket https://console.cloud.google.com/storage/browser/shuffler.appspot.com?project=shuffler -4. Go to the functions https://console.cloud.google.com/functions/list?project=shuffler - - -## How it works (from frontend to backend) -### Choose mailfolders -1. Use microsoft graph api to get the folders the user wants to listen to -* Have the user write their primary email (default) or another one -* Have it show the folders for the email with chooseable buttons somehow -|inbox -|-subinbox -|--subsubinbox <-- choose e.g. this one -|otherfolder - -API: -// requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/me/mailfolders") - -### Add callback subscription -2. Make an APIcall to ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") with callback url defined as "https://shuffler.io/api/v1/workflows/{key}/email/authorize" -* Should this be set up whenever the user clicked start or when the workflow is created? -* Start click -> -1. Add another cloud function for the item -2. When it's ready, deploy it to authorize -3. Show it as ready - -### Remove a subscription -* Since everything is already generated above, one would need to -* https://docs.microsoft.com/en-us/graph/api/subscription-delete?view=graph-rest-1.0&tabs=http -* DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} - - -## CREATE - Fixme: LIST all current subscriptions, and stop them if they're towards the same endpoint -* POST /api/v1/workflows/{key}/outlook -* createOutlookSub(resp, request) -* getOutlookSubscriptions(client) // Used to remove all existing for same endpoint -* makeOutlookSubscription(client, folderIds, notificationUrl) -* Add data from ^ to triggerAuth - -## DELETE -* DELETE /api/v1/workflows/{key}/outlook/{triggerId} -* handleDeleteOutlookSub(resp, request) -* handleOutlookSubRemoval(workflowId, triggerId) diff --git a/functions/triggers/outlook/hook.go b/functions/triggers/outlook/hook.go deleted file mode 100644 index 9762c2c2..00000000 --- a/functions/triggers/outlook/hook.go +++ /dev/null @@ -1,222 +0,0 @@ -package function - -// Shuffle: -// https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/Authentication/appId/e080cbf4-5dba-44b4-8643-a7c982189c16/isMSAApp//defaultBlade/Overview/servicePrincipalCreated/true - -// Oauth playground: -// https://oauthplay.azurewebsites.net/ - -// APPS: -// https://apps.dev.microsoft.com - -// REMOVE ACCESS: -// https://portal.office.com/account/# - -// Developer: -// https://developer.microsoft.com/en-us/graph/docs/concepts/permissions_reference - -// Bots: -// https://dev.botframework.com/bots - -// Connectors -// https://outlook.office.com/connectors/home/login/#/new -// https://go.microsoft.com/fwlink/?linkid=857599 - -import ( - //"encoding/json" - "bytes" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "time" -) - -type Info struct { - Url string `json:"url" datastore:"url"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` -} - -// Actions to be done by webhooks etc -// Field is the actual field to use from json -type HookAction struct { - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Id string `json:"id" datastore:"id"` - Field string `json:"field" datastore:"field"` -} - -type Hook struct { - Id string `json:"id" datastore:"id"` - Info Info `json:"info" datastore:"info"` - Actions []HookAction `json:"actions" datastore:"actions"` - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Running bool `json:"running" datastore:"running"` -} - -type TeamsHook struct { - MembersAdded []struct { - ID string `json:"id"` - } `json:"membersAdded"` - Type string `json:"type"` - Timestamp time.Time `json:"timestamp"` - LocalTimestamp string `json:"localTimestamp"` - ID string `json:"id"` - ChannelID string `json:"channelId"` - ServiceURL string `json:"serviceUrl"` - From struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"from"` - Conversation struct { - IsGroup bool `json:"isGroup"` - ConversationType string `json:"conversationType"` - ID string `json:"id"` - TenantID string `json:"tenantId"` - } `json:"conversation"` - Recipient struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"recipient"` - ChannelData struct { - Team struct { - ID string `json:"id"` - } `json:"team"` - EventType string `json:"eventType"` - Tenant struct { - ID string `json:"id"` - } `json:"tenant"` - } `json:"channelData"` -} - -var hook Hook -var baseUrl = "https://shuffler.io" - -type OauthToken struct { - TokenType string `json:"token_type"` - ExpiresIn int `json:"expires_in"` - ExtExpiresIn int `json:"ext_expires_in"` - AccessToken string `json:"access_token"` -} - -type TeamsResponse struct { - Conversation struct { - ID string `json:"id"` - } `json:"conversation"` - From struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"from"` - Recipient struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"recipient"` - ReplyToId string `json:"replyToId"` - Type string `json:"type"` - Text string `json:"text"` -} - -type O365hook struct { - OdataContext string `json:"@odata.context"` - Value []struct { - OdataType string `json:"@odata.type"` - ID interface{} `json:"Id"` - SubscriptionID string `json:"SubscriptionId"` - SubscriptionExpirationDateTime time.Time `json:"SubscriptionExpirationDateTime"` - SequenceNumber int `json:"SequenceNumber"` - ChangeType string `json:"ChangeType"` - Resource string `json:"Resource"` - ResourceData struct { - OdataType string `json:"@odata.type"` - OdataID string `json:"@odata.id"` - OdataEtag string `json:"@odata.etag"` - ID string `json:"Id"` - } `json:"ResourceData"` - } `json:"value"` -} - -func Authorization(resp http.ResponseWriter, request *http.Request) { - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body: %s", err) - resp.WriteHeader(403) - return - } - - if len(body) > 0 { - // In here - get the email data - // Check who it belongs to and run those workflows - // This should be set from run.go with the callback - // userId: {subscriptionID: {workflowID}} - // E.g. workflow: {auth: {userId - // workflow: {trigger: { - - err = forwardRequest(body) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(403) - return - } - - resp.WriteHeader(200) - resp.Write([]byte("OK")) - return - } - - token := request.URL.Query().Get("validationToken") - if len(token) == 0 { - log.Println("Validation token is missing") - resp.WriteHeader(403) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(string(token))) -} - -// GetUserDetails - Get one user's details from randomuser.me API -func forwardRequest(body []byte) error { - callbackUrl := os.Getenv("CALLBACKURL") - workflowId := os.Getenv("WORKFLOW_ID") - apikey := os.Getenv("FUNCTION_APIKEY") - - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, workflowId) - //log.Printf("Sending data to %s", fullUrl) - - data := fmt.Sprintf(`{"execution_argument": "%s"}`, string(body)) - - req, err := http.NewRequest( - http.MethodPost, - fullUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - return err - } - - req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey)) - req.Header.Add("Content-Type", "application/json") - randomUserClient := http.Client{ - Timeout: time.Second * 5, - } - - res, err := randomUserClient.Do(req) - if err != nil { - return err - } - - log.Printf("Status: %d", res.StatusCode) - returnbody, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - - log.Printf("New body: %s", string(returnbody)) - - //log.Println(string(newbody)) - return nil -} diff --git a/functions/triggers/outlook/integrations/config.json b/functions/triggers/outlook/integrations/config.json deleted file mode 100644 index 8d98dba3..00000000 --- a/functions/triggers/outlook/integrations/config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "clientID": "70e37005-c954-4290-b573-d4b94e484336", - "clientSecret": ".eNw/A[kQFB5zL.agvRputdEJENeJ392", - "RedirectURL": "https://44d84ee7.ngrok.io/functions/outlook/register", - "AuthURL": "https://login.microsoftonline.com/common/oauth2/authorize", - "TokenURL": "https://login.microsoftonline.com/common/oauth2/token" -} diff --git a/functions/triggers/outlook/integrations/server.crt b/functions/triggers/outlook/integrations/server.crt deleted file mode 100644 index 92244399..00000000 --- a/functions/triggers/outlook/integrations/server.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDYDCCAkigAwIBAgIJAOvgxcclM1eyMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV -BAYTAk5PMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQwHhcNMTgwMzA1MTE1MTIwWhcNMjgwMzAyMTE1MTIwWjBF -MQswCQYDVQQGEwJOTzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAowNqrvocJCLLcytYBfZhsqG3CkPsJ4PTicNyjuUGmlILPHlN1DE7jbDo -KuOKImfV4AfQANnttaPksZyuKJL8XGpC7YmF5mmInUWG48SmZjvRbBi1LnCrjJKS -ywh2lJqw4w30w2ItcpogDrYhh6+T3VyabcYngZjKSFgON3wo4I0c6aT19VVXGqnK -y1WEejZmiChV4iwEu4vMPIzt16QpIBr8NPSkBLLRDAGWMjFnIuYDEwgjVn6XYhM9 -+NxhvY9es+qeqLQsZj2a1wcDGaw7G4iNZdltlPmlTCreRDBYBTsYCds/rJmPZ2xi -6jgiyNZj3xHG5Knw2YIw0OwHyT9mWQIDAQABo1MwUTAdBgNVHQ4EFgQUckr5tCzg -F1eBID7mtWTfeqyX4g0wHwYDVR0jBBgwFoAUckr5tCzgF1eBID7mtWTfeqyX4g0w -DwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAdH1aPcWBjJHF6n/ -NgIiRSEU4mi5I6RUlPuR7dN7dcmF1Ho7quurNFzknXwks+a62oKSnkkFxxwrv/d6 -dIH5kNibVs7oRxEpA3gUmXXKUW4RPrxAp2zN37t7zs5xbpTATxfJiIMP8Rjo0sOf -SilS22Sn0e0HxBi78t3DJEZvOQ9KSRuD1g9gOAY4lj/fni6rVJo8YCR2MyjmQoXB -luHDF4jTqi/TkXECfqQZu0pctx3maISpB1fAuaELwPDvqbLgoC97gl6bIEiKLkJ0 -JkbGnb997K80ztFvFAKGyUtsvEDCupe/fdBPFqCruAQWI/BqVJFTdRD43dWEQANF -S8ApvA== ------END CERTIFICATE----- diff --git a/functions/triggers/outlook/integrations/server.key b/functions/triggers/outlook/integrations/server.key deleted file mode 100644 index 87c60e08..00000000 --- a/functions/triggers/outlook/integrations/server.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAowNqrvocJCLLcytYBfZhsqG3CkPsJ4PTicNyjuUGmlILPHlN -1DE7jbDoKuOKImfV4AfQANnttaPksZyuKJL8XGpC7YmF5mmInUWG48SmZjvRbBi1 -LnCrjJKSywh2lJqw4w30w2ItcpogDrYhh6+T3VyabcYngZjKSFgON3wo4I0c6aT1 -9VVXGqnKy1WEejZmiChV4iwEu4vMPIzt16QpIBr8NPSkBLLRDAGWMjFnIuYDEwgj -Vn6XYhM9+NxhvY9es+qeqLQsZj2a1wcDGaw7G4iNZdltlPmlTCreRDBYBTsYCds/ -rJmPZ2xi6jgiyNZj3xHG5Knw2YIw0OwHyT9mWQIDAQABAoIBABJ+9L/dyQuglwz+ -QgKLLhKinq4fftAM+ReMgZcNDW69GGFIMjh9TZCKHg2fu7Cjr3S37jXqhDoz2mL8 -sBYSd2fU9rsU+4hlOQb/OIrnaSn4Z46oTwZx6kUM7HL1Bt9dnexlTPxOS3HRYwnI -SI2oslJPi4YhEaJ2v5ztwM8y20B/E/zSW2onWz5gB8/bdxSmuJaWfHioIEoac8Gf -BE7jiYMnx9kKeVfgkKPMBKXhAyE2lbAz7N5nDS/4HkbUMh389RBvakc4gkv/QkSW -LNuXpbcSqJiG0FcVutjYS87a/ul3IdhAYmZDTuvRUhNsWBiY3LSY4C9NlHhiJihg -RU1kknECgYEA1dPT6n4hy/OK2mT3ThGQQFMJWsnmcHSsvuK31+UYZOI9kNtBVxc6 -HSHkh0G53o6o2wZLr2gMXy35O5ZxSbectA1Q4MJDlYBs1MfHdOVyE4z6mmA9TLN1 -c+9pYOC0qx+6NwPdO7j2xdUMEUfzocWOeay0AzJg20BQYmKuy3Kc+tcCgYEAwyn6 -n+XS0vodfJdHhvbW/jocQlHQOBK5HklZfq2PMgpRRDaBOvHP/f07egLc2inec2sC -yPSCEfRMMhFcU5NoBt4Unzz2Y8pbpL1L5kbM6B4IqK/5vYcbvkmBkhL3AFT6Fg/3 -3XCdygPW9Vf1nRKr2KhT9dDvB+XO2B75JmKwMk8CgYApKzGf8kz7gZZ4WfwrccI+ -QD6K1lihyjUAQ5J15Mv/kHeeDjjUVcqAlWf0irkImpr0IJAt43COWsGjsWF6efmX -yQCLZZuxixppFVXXsd122ivd0S28OMkiWzQEzP67+83Ujc/okcIhcNVz9lB4ExtN -Xe0CuI5haE6RwsI4tYZ33QKBgFq6ckPRcOAZ3IlmPp9Us3/+fdKq/BSFR7/3s347 -q11FBKCkghFoBxx5lCPVntxhKIQZlHLdkHZOTvnbrkNAPNUsewPIMHcVxOLiCZ3k -/i9OfxIEtSJR5CjjPTQuUtu5pYWKKN2uE/ytKkpmeM1rt64CGv4lAmp2gGFijMs2 -h9jrAoGBANPQO6cKqtnxvst3lnljVBoftlJgeHamUac+xeYKA5Hocv5VwLXMTzzu -09tAhQFvFwCWWrfdgtvIM6k5Sl9F5MdiO9VNflI0IVudIcm9FKorWogH02mtwxsw -hvk5VUk3awiZ/Nu9t38ukeqCetjQEf6yupy/14ZPLndN5naSeEwo ------END RSA PRIVATE KEY----- diff --git a/functions/triggers/outlook/main.go b/functions/triggers/outlook/main.go deleted file mode 100644 index 12bc84b3..00000000 --- a/functions/triggers/outlook/main.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "log" - "net/http" - "os" - - "github.com/gorilla/handlers" - "github.com/gorilla/mux" -) - -func webhook() { - // FIXME - remove static - port := ":8080" - baseFilePath := "/" - - mux := mux.NewRouter() - mux.SkipClean(true) - - // FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend - mux.HandleFunc(baseFilePath, Authorization).Methods("POST") - mux.HandleFunc("/authorize", Authorization).Methods("POST") - - handlers.LoggingHandler(os.Stdout, mux) - loggedRouter := handlers.LoggingHandler(os.Stdout, mux) - - log.Printf("Starting on http://localhost%s", port) - err := http.ListenAndServe( - port, - loggedRouter, - ) - - if err != nil { - log.Fatal("ListenAndServer: ", err) - } - -} - -func main() { - webhook() -} diff --git a/functions/triggers/outlook/run.go b/functions/triggers/outlook/run.go deleted file mode 100644 index 55d3d420..00000000 --- a/functions/triggers/outlook/run.go +++ /dev/null @@ -1,304 +0,0 @@ -package main - -// This entire script should be part of the API backend - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "time" - - "golang.org/x/oauth2" -) - -type Subscription struct { - ChangeType string `json:"changeType"` - NotificationURL string `json:"notificationUrl"` - Resource string `json:"resource"` - ExpirationDateTime string `json:"expirationDateTime"` - ClientState string `json:"clientState"` -} - -// ClientState string `json:"ClientState,omitempty"` -// OdataType string `json:"@odata.type"` - -//odata.type - Include "@odata.type":"#Microsoft.OutlookServices.PushSubscription". The PushSubscription entity defines NotificationURL. -//ChangeType - Specifies the types of events to monitor for that resource. See ChangeType for the supported types. -//ClientState - Optional property that indicates that each notification should be sent with a header by the same ClientState value. This lets the listener check the legitimacy of each notification. -//NotificationURL - Specifies where notifications should be sent to. This URL represents a web service typically implemented by the client. -//Resource - Specifies the resource to monitor and receive notifications on. You can use the optional query parameter $filter to refine the conditions for a notification, or use $select to include specific properties in a rich notification. - -//https://outlook.office.com/mail.read - -type Config struct { - ClientID string - ClientSecret string - RedirectUrl string - AuthUrl string - TokenUrl string -} - -func getOfficeAppInfo() (Config, error) { - configpath := "integrations/config.json" - - data, err := ioutil.ReadFile(configpath) - if err != nil { - //log.Fatal(err) - log.Printf("Error getting hive: %s\n", err) - } - - config := Config{} - err = json.Unmarshal(data, &config) - if err != nil { - return Config{}, err - } - - return config, nil -} - -// This should be a popup for the user -func get_accesstoken() (*http.Client, OauthToken, error) { - ctx := context.Background() - config, err := getOfficeAppInfo() - if err != nil { - return nil, OauthToken{}, err - } - - conf := &oauth2.Config{ - ClientID: config.ClientID, - ClientSecret: config.ClientSecret, - Scopes: []string{ - "Mail.Read", - "User.Read", - "https://outlook.office.com/mail.read", - }, - RedirectURL: "https://localhost:8000", - Endpoint: oauth2.Endpoint{ - AuthURL: config.AuthUrl, - TokenURL: config.TokenUrl, - }, - } - //"Mail.Read.Shared", - - //url := conf.AuthCodeURL("state", oauth2.SetAuthURLParam("resource", "https://outlook.office.com")) - // ADD DATA TO STATE HERE :O - url := conf.AuthCodeURL("workflow_id%3Dc2e0b50a-2957-427e-a97b-b989dc5a5408%26trigger_id%3D9e845679-5843-4959-a76c-a6d664e9df35%26username%3Drheyix.yt@gmail.com", oauth2.SetAuthURLParam("resource", "https://graph.microsoft.com")) - - fmt.Printf("Visit the URL for the auth dialog: \n%v\n\n", url) - codechannel := make(chan string) - - // Handles the server callback, listening on port 8000 - go func() { - port := ":8000" - - http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) { - tmpcode := request.URL.Query().Get("code") - if len(tmpcode) < 100 { - return - } else { - codechannel <- tmpcode - } - }) - - // FIX - might cause errors not being printed - err := http.ListenAndServeTLS(port, "integrations/server.crt", "integrations/server.key", nil) - if err != nil { - log.Printf("%s\n", err) - } - }() - - code := <-codechannel - close(codechannel) - - // https://stackoverflow.com/questions/52787420/multiple-resources-in-a-single-authorization-request - // Multi resource ^ - access_token, err := conf.Exchange(ctx, code) - //log.Printf("%#v", access_token) - if err != nil { - return nil, OauthToken{}, err - } - - //log.Printf("%#v", access_token) - outlookClient := conf.Client(ctx, access_token) - - oauthToken := OauthToken{ - AccessToken: access_token.AccessToken, - TokenType: access_token.TokenType, - RefreshToken: access_token.RefreshToken, - Expiry: access_token.Expiry, - } - - return outlookClient, oauthToken, nil -} - -type OauthToken struct { - AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` - TokenType string `json:"TokenType" datastore:"TokenType,noindex"` - RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` - Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` -} - -type Mailfolders struct { - OdataContext string `json:"@odata.context"` - OdataNextLink string `json:"@odata.nextLink"` - Value []struct { - ID string `json:"id"` - DisplayName string `json:"displayName"` - ParentFolderID string `json:"parentFolderId"` - ChildFolderCount int `json:"childFolderCount"` - UnreadItemCount int `json:"unreadItemCount"` - TotalItemCount int `json:"totalItemCount"` - } `json:"value"` -} - -func getFolders(client *http.Client) (Mailfolders, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/frikky@shuffletest.onmicrosoft.com/mailfolders") - - ret, err := client.Get(requestUrl) - if err != nil { - log.Printf("FolderErr: %s", err) - return Mailfolders{}, err - } - - log.Printf("Status folders: %d", ret.StatusCode) - body, err := ioutil.ReadAll(ret.Body) - if err != nil { - log.Printf("Body: %s", err) - return Mailfolders{}, err - } - - //log.Printf("Body: %s", string(body)) - - mailfolders := Mailfolders{} - err = json.Unmarshal(body, &mailfolders) - if err != nil { - log.Printf("Unmarshal: %s", err) - return Mailfolders{}, err - } - - //fmt.Printf("%#v", mailfolders) - // FIXME - recursion for subfolders - // Recursive struct - // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) - for _, folder := range mailfolders.Value { - log.Println(folder.DisplayName) - } - - return mailfolders, nil -} - -// Subscribes to a mailbox based on some thingies -func makeSubscription(client *http.Client, folderIds []string) { - // FIXME - show the users folders from oauth and let them choose - - fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" - //resource := fmt.Sprintf("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") - resource := fmt.Sprintf("me/mailfolders('inbox')/messages") - sub := Subscription{ - ChangeType: "created", - NotificationURL: "https://de4fc12b.ngrok.io", - ExpirationDateTime: "2019-09-22T18:23:45.9356913Z", - ClientState: "This is a test", - Resource: resource, - } - - data, err := json.Marshal(sub) - if err != nil { - log.Printf("Marshal: %s", err) - return - } - - log.Printf(string(data)) - - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer(data), - ) - req.Header.Add("Content-Type", "application/json") - - res, err := client.Do(req) - if err != nil { - log.Printf("Client: %s", err) - return - } - - log.Printf("Status: %d", res.StatusCode) - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Body: %s", err) - return - } - - fmt.Println(string(body)) - log.Println("Shoooould be set up :)") - -} - -func getOutlookClient(code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { - ctx := context.Background() - - conf := &oauth2.Config{ - ClientID: "70e37005-c954-4290-b573-d4b94e484336", - ClientSecret: ".eNw/A[kQFB5zL.agvRputdEJENeJ392", - Scopes: []string{ - "Mail.Read", - "User.Read", - "https://outlook.office.com/mail.read", - }, - RedirectURL: redirectUri, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", - TokenURL: "https://login.microsoftonline.com/common/oauth2/token", - }, - } - - if len(code) > 0 { - access_token, err := conf.Exchange(ctx, code) - if err != nil { - log.Printf("Access_token issue: %s", err) - return &http.Client{}, access_token, err - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } else { - // Manually recreate the oauthtoken - access_token := &oauth2.Token{ - AccessToken: accessToken.AccessToken, - TokenType: accessToken.TokenType, - RefreshToken: accessToken.RefreshToken, - Expiry: accessToken.Expiry, - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } -} - -func main() { - graphclient, oauthToken, err := get_accesstoken() - if err != nil { - log.Printf("Oauth setup: %s", err) - return - } - - // FIXME - make this possible for alternative users (shared) - folders, err := getFolders(graphclient) - if err != nil { - log.Printf("Folder get error: %s", err) - return - } - _ = folders - - folderIds := []string{"inbox"} - //log.Println(folders) - //log.Printf("%#v", oauthToken) - // Use oauthToken to generate data for outlook - outlookclient, _, err := getOutlookClient("", oauthToken, "https://localhost:8000") - makeSubscription(outlookclient, folderIds) -} From 4efaf4c75423d700480b40832f0c77ca4b2f0d4e Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 May 2020 20:06:56 +0200 Subject: [PATCH 02/13] Reimplemented app_sdk for code gen --- backend/go-app/main.go | 151 +++++++++++++++++++++- backend/go-app/walkoff.go | 108 ++++++++++++++++ frontend/src/App.js | 6 +- functions/onprem/app_sdk/Dockerfile | 16 --- functions/onprem/app_sdk/LICENSE | 21 +++ functions/onprem/app_sdk/README.md | 12 +- functions/onprem/app_sdk/build.sh | 4 - functions/onprem/app_sdk/requirements.txt | 2 - functions/triggers/msteams/hook.go | 1 + 9 files changed, 285 insertions(+), 36 deletions(-) delete mode 100644 functions/onprem/app_sdk/Dockerfile create mode 100644 functions/onprem/app_sdk/LICENSE delete mode 100644 functions/onprem/app_sdk/build.sh delete mode 100644 functions/onprem/app_sdk/requirements.txt diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3b413141..294c8bb7 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5148,6 +5148,135 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { resp.Write(urlbody) } +func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { + type versionCheck struct { + Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` + SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"` + OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"` + } + + //body = []byte(`swagger: "2.0"`) + //body = []byte(`swagger: '1.0'`) + //newbody := string(body) + //newbody = strings.TrimSpace(newbody) + //body = []byte(newbody) + //log.Println(string(body)) + //tmpbody, err := yaml.YAMLToJSON(body) + //log.Println(err) + //log.Println(string(tmpbody)) + + // This has to be done in a weird way because Datastore doesn't + // support map[string]interface and similar (openapi3.Swagger) + var version versionCheck + + parsed := ParsedOpenApi{} + swaggerdata := []byte{} + idstring := "" + + isJson := false + err := json.Unmarshal(body, &version) + if err != nil { + //log.Printf("Json err: %s", err) + err = yaml.Unmarshal(body, &version) + if err != nil { + log.Printf("Yaml error: %s", err) + } else { + log.Printf("Successfully parsed YAML!") + } + } else { + isJson = true + log.Printf("Successfully parsed JSON!") + } + + if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { + version.Swagger = version.SwaggerVersion + } + + if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { + log.Println("Handling v3 API") + swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + if err != nil { + return ParsedOpenApi{}, err + } + + hasher := md5.New() + hasher.Write(body) + idstring = hex.EncodeToString(hasher.Sum(nil)) + + log.Printf("Swagger v3 validation success with ID %s!", idstring) + log.Printf("Paths: %d", len(swagger.Paths)) + + if !isJson { + log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) + } + + log.Printf("Body: %s", string(swaggerdata)) + + //return nil + //return ParsedOpenApi{}, err + } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { + // Convert + log.Println("Handling v2 API") + var swagger openapi2.Swagger + //log.Println(string(body)) + err = json.Unmarshal(body, &swagger) + if err != nil { + //log.Printf("Json error? %s", err) + err = gyaml.Unmarshal(body, &swagger) + if err != nil { + log.Printf("Yaml error: %s", err) + return ParsedOpenApi{}, err + } else { + log.Printf("Valid yaml!") + } + + } + + swaggerv3, err := openapi2conv.ToV3Swagger(&swagger) + if err != nil { + log.Printf("Failed converting from openapi2 to 3: %s", err) + return ParsedOpenApi{}, err + } + + swaggerdata, err = json.Marshal(swaggerv3) + if err != nil { + log.Printf("Failed unmarshaling v3 data: %s", err) + return ParsedOpenApi{}, err + } + + hasher := md5.New() + hasher.Write(swaggerdata) + idstring = hex.EncodeToString(hasher.Sum(nil)) + if !isJson { + log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring) + } + log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring) + + ctx := context.Background() + err = setOpenApiDatastore(ctx, idstring, parsed) + if err != nil { + log.Printf("Failed uploading openapi2 to datastore: %s", err) + return ParsedOpenApi{}, err + } + + } + + log.Printf("Down here?") + if len(swaggerdata) > 0 { + body = swaggerdata + } + + // Parsing it to swagger 3 + parsed = ParsedOpenApi{ + ID: idstring, + Body: string(body), + Success: true, + } + + return parsed, err +} + +// FIXME: Migrate this to use handleSwaggerValidation() func validateSwagger(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5656,7 +5785,7 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed getting apps: %s", err) } else if err == nil && len(workflowapps) == 0 { - log.Printf("Apps: loading TEST") + log.Printf("Downloading default workflow apps") fs := memfs.New() storer := memory.NewStorage() @@ -5696,6 +5825,26 @@ func runInit(ctx context.Context) { iterateAppGithubFolders(fs, dir, "", "") } + log.Printf("Downloading OpenAPI data for search - EXTRA APPS") + apis := "https://github.com/frikky/OpenAPI-security-definitions" + fs := memfs.New() + storer := memory.NewStorage() + cloneOptions := &git.CloneOptions{ + URL: apis, + } + _, err = git.Clone(storer, fs, cloneOptions) + if err != nil { + log.Printf("Failed loading repo %s into memory: %s", err) + } else { + dir, err := fs.ReadDir("") + if err != nil { + log.Printf("Failed reading folder: %s", err) + } + + iterateOpenApiGithub(fs, dir, "", "") + log.Printf("Finished downloading extra API samples") + } + log.Printf("Finished INIT") } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 9c039655..7234c6b7 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -29,6 +29,7 @@ import ( http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" newscheduler "github.com/carlescere/scheduler" + "github.com/getkin/kin-openapi/openapi3" "github.com/go-git/go-git/v5/storage/memory" //"github.com/gorilla/websocket" //"google.golang.org/appengine" @@ -3257,6 +3258,113 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { + for _, file := range dir { + if len(onlyname) > 0 && file.Name() != onlyname { + continue + } + + // Folder? + switch mode := file.Mode(); { + case mode.IsDir(): + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + dir, err := fs.ReadDir(tmpExtra) + if err != nil { + log.Printf("Failed to read dir: %s", err) + break + } + + // Go routine? Hmm, this can be super quick I guess + err = iterateOpenApiGithub(fs, dir, tmpExtra, "") + if err != nil { + break + } + case mode.IsRegular(): + // Check the file + filename := file.Name() + if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") { + //log.Printf("Found file: %s", filename) + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + + fileReader, err := fs.Open(tmpExtra) + if err != nil { + continue + } + + readFile, err := ioutil.ReadAll(fileReader) + if err != nil { + log.Printf("Filereader error yaml: %s", err) + continue + } + + // 1. This parses OpenAPI v2 to v3 etc, for use. + parsedOpenApi, err := handleSwaggerValidation(readFile) + if err != nil { + log.Printf("Validation error: %s", err) + continue + } + + log.Printf("%#v", parsedOpenApi) + + // 2. With parsedOpenApi.ID: + //http://localhost:3000/apps/new?id=06b1376f77b0563a3b1747a3a1253e88 + + // 3. Load this as a "standby" app + // FIXME: This should be a function ROFL + //log.Printf("%s", string(readFile)) + swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body)) + if err != nil { + log.Printf("Swagger validation error in loop: %s", err) + continue + } + + if strings.Contains(swagger.Info.Title, " ") { + strings.Replace(swagger.Info.Title, " ", "", -1) + } + + basePath, err := buildStructure(swagger, parsedOpenApi.ID) + if err != nil { + log.Printf("Failed to build base structure in loop: %s", err) + continue + } + + log.Printf("Should generate yaml") + api, pythonfunctions, err := generateYaml(swagger, parsedOpenApi.ID) + if err != nil { + log.Printf("Failed building and generating yaml in loop: %s", err) + continue + } + + // FIXME: Configure user? + api.Owner = "" + err = dumpApi(basePath, api) + if err != nil { + log.Printf("Failed dumping yaml in loop: %s", err) + continue + } + + // FIXME: Should this continue on activation? + + identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, parsedOpenApi.ID) + classname := strings.Replace(identifier, " ", "", -1) + classname = strings.Replace(classname, "-", "", -1) + parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) + if err != nil { + log.Printf("Failed dumping python in loop: %s", err) + continue + } + + identifier = strings.Replace(identifier, " ", "-", -1) + identifier = strings.Replace(identifier, "_", "-", -1) + + return nil + } + } + } + + return nil +} + // Onlyname is used to func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { var err error diff --git a/frontend/src/App.js b/frontend/src/App.js index a492c501..63c4f860 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -38,12 +38,10 @@ import AlertTemplate from "react-alert-template-basic"; import { positions, Provider } from "react-alert"; // Testing - localhost -//const globalUrl = "http://192.168.3.6:5001" -//console.log("HOST: ", process.env) - +const globalUrl = "http://192.168.3.6:5001" // Production - backend proxy forwarding in nginx -const globalUrl = window.location.origin +//const globalUrl = window.location.origin const surfaceColor = "#27292D" const inputColor = "#383B40" diff --git a/functions/onprem/app_sdk/Dockerfile b/functions/onprem/app_sdk/Dockerfile deleted file mode 100644 index 44294a5b..00000000 --- a/functions/onprem/app_sdk/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.7-alpine as base - -FROM base as builder -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/functions/onprem/app_sdk/LICENSE b/functions/onprem/app_sdk/LICENSE new file mode 100644 index 00000000..ce11f6f3 --- /dev/null +++ b/functions/onprem/app_sdk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Frikkylikeme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/functions/onprem/app_sdk/README.md b/functions/onprem/app_sdk/README.md index a68aa39d..873d45d3 100644 --- a/functions/onprem/app_sdk/README.md +++ b/functions/onprem/app_sdk/README.md @@ -1,10 +1,4 @@ -# app_sdk -This is the SDK used for apps to behave like they should. -To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. +# App_sdk +App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk -## If you want to update apps.. PS: downloads from docker hub do overrides.. :) -1. Write your code & check if runtime works -2. Build app_base image -3. docker rm $(docker ps -aq) # Remove all stopped containers -4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) -5. Rebuild the Docker image (click load in GUI?) +It's under MIT license, NOT AGPLv3. diff --git a/functions/onprem/app_sdk/build.sh b/functions/onprem/app_sdk/build.sh deleted file mode 100644 index c93c83fc..00000000 --- a/functions/onprem/app_sdk/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -docker rmi frikky/shuffle:app_sdk -docker build . -t frikky/shuffle:app_sdk --no-cache -docker push frikky/shuffle:app_sdk diff --git a/functions/onprem/app_sdk/requirements.txt b/functions/onprem/app_sdk/requirements.txt deleted file mode 100644 index 804abb1b..00000000 --- a/functions/onprem/app_sdk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -urllib3 diff --git a/functions/triggers/msteams/hook.go b/functions/triggers/msteams/hook.go index fae0e594..5a3a0b0d 100644 --- a/functions/triggers/msteams/hook.go +++ b/functions/triggers/msteams/hook.go @@ -168,6 +168,7 @@ func sendRequest(token OauthToken, message TeamsHook) error { return nil } +// If you're finding this: its from a test project :) func get_accesstoken() (OauthToken, error) { client_id := "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f" client_secret := ":3]D6oFimiXbuV20xH?Dzu@LR*6IFVbq" From 8fd2f9df0ccc2d77d5e2e5484e67a09497db907e Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 May 2020 20:36:57 +0200 Subject: [PATCH 03/13] Initial backend app search engine finished --- backend/go-app/codegen.go | 9 +++-- backend/go-app/main.go | 3 -- backend/go-app/walkoff.go | 76 +++++++++++++++++++++++++-------------- frontend/src/Apps.js | 5 +++ 4 files changed, 60 insertions(+), 33 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index e737c426..2d4f09bc 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -773,8 +773,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor Parameters: extraParameters, } - log.Printf("FUNCTION: %#v", action) - action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) @@ -791,6 +789,11 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalParameters := []WorkflowAppActionParameter{} if len(path.Get.Parameters) > 0 { for _, param := range path.Get.Parameters { + if param.Value.Schema.Ref == "" { + continue + } + + log.Printf("TYPE: %#v", param.Value.Schema) curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1013,7 +1016,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string, firstQuery bool) (WorkflowAppAction, string) { // What to do with this, hmm - log.Printf("PATH: %s", actualPath) + //log.Printf("PATH: %s", actualPath) functionName := fixFunctionName(path.Post.Summary, actualPath) action := WorkflowAppAction{ diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 294c8bb7..80449235 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5210,8 +5210,6 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) } - log.Printf("Body: %s", string(swaggerdata)) - //return nil //return ParsedOpenApi{}, err } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { @@ -5261,7 +5259,6 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { } - log.Printf("Down here?") if len(swaggerdata) > 0 { body = swaggerdata } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 7234c6b7..6e0de95a 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -86,10 +86,11 @@ type WorkflowApp struct { Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` + Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` Owner string `json:"owner" datastore:"owner" yaml:"owner"` PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description" required:false yaml:"description"` + Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` @@ -2918,6 +2919,7 @@ func handleGetfile(resp http.ResponseWriter, request *http.Request) ([]byte, err return buf.Bytes(), nil } +// Basically a search for apps that aren't activated yet func getSpecificApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2958,9 +2960,41 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D log.Printf("Body: %s", string(body)) + ctx := context.Background() + workflowapps, err := getAllWorkflowApps(ctx) + if err != nil { + log.Printf("Error: Failed getting workflowapps: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + returnValues := []WorkflowApp{} + search := strings.ToLower(tmpBody.Search) + for _, app := range workflowapps { + if !app.Activated && app.Generated { + // This might be heavy with A LOT + // Not too worried with todays tech tbh.. + appName := strings.ToLower(app.Name) + appDesc := strings.ToLower(app.Description) + if strings.Contains(appName, search) || strings.Contains(appDesc, search) { + log.Printf("Name: %s, Generated: %s, Activated: %s", app.Name, strconv.FormatBool(app.Generated), strconv.FormatBool(app.Activated)) + returnValues = append(returnValues, app) + } + } + } + + newbody, err := json.Marshal(returnValues) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow executions"}`))) + return + } + + returnData := fmt.Sprintf(`{"success": true, "reason": %s}`, string(newbody)) + log.Printf("Return: %s", returnData) resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + resp.Write([]byte(returnData)) } func validateAppInput(resp http.ResponseWriter, request *http.Request) { @@ -3283,6 +3317,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, // Check the file filename := file.Name() if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") { + log.Printf("File: %s", filename) //log.Printf("Found file: %s", filename) tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) @@ -3304,8 +3339,6 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, continue } - log.Printf("%#v", parsedOpenApi) - // 2. With parsedOpenApi.ID: //http://localhost:3000/apps/new?id=06b1376f77b0563a3b1747a3a1253e88 @@ -3322,14 +3355,8 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, strings.Replace(swagger.Info.Title, " ", "", -1) } - basePath, err := buildStructure(swagger, parsedOpenApi.ID) - if err != nil { - log.Printf("Failed to build base structure in loop: %s", err) - continue - } - log.Printf("Should generate yaml") - api, pythonfunctions, err := generateYaml(swagger, parsedOpenApi.ID) + api, _, err := generateYaml(swagger, parsedOpenApi.ID) if err != nil { log.Printf("Failed building and generating yaml in loop: %s", err) continue @@ -3337,26 +3364,20 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, // FIXME: Configure user? api.Owner = "" - err = dumpApi(basePath, api) + api.ID = parsedOpenApi.ID + api.IsValid = true + api.Generated = true + api.Activated = false + + ctx := context.Background() + err = setWorkflowAppDatastore(ctx, api, api.ID) if err != nil { - log.Printf("Failed dumping yaml in loop: %s", err) + log.Printf("Failed setting workflowapp in loop: %s", err) continue + } else { + log.Printf("Added %s:%s to the database from OpenAPI repo", api.Name, api.AppVersion) } - // FIXME: Should this continue on activation? - - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, parsedOpenApi.ID) - classname := strings.Replace(identifier, " ", "", -1) - classname = strings.Replace(classname, "-", "", -1) - parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) - if err != nil { - log.Printf("Failed dumping python in loop: %s", err) - continue - } - - identifier = strings.Replace(identifier, " ", "-", -1) - identifier = strings.Replace(identifier, "_", "-", -1) - return nil } } @@ -3583,6 +3604,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { workflowapp.ID = uuid.NewV4().String() workflowapp.IsValid = true workflowapp.Generated = false + workflowapp.Activated = true err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index 7b343e0d..b4bfb87c 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -652,6 +652,11 @@ const Apps = (props) => { }) .then((responseJson) => { console.log(responseJson) + if (responseJson.success) { + if (responseJson.reason !== undefined && responseJson.reason.length > 0) { + setFilteredApps(responseJson.reason) + } + } }) .catch(error => { alert.error(error.toString()) From ccf5091fc7391cdbd0582c1ec64c2b9f11a8998a Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 May 2020 20:59:52 +0200 Subject: [PATCH 04/13] Fixed preloading of some apps --- backend/go-app/codegen.go | 10 ++++---- backend/go-app/main.go | 22 ++++++++-------- backend/go-app/walkoff.go | 54 ++++++++++++++++++++++++++++----------- frontend/src/Apps.js | 4 +-- 4 files changed, 57 insertions(+), 33 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 2d4f09bc..889c20e7 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -366,7 +366,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri securitySchemes := swagger.Components.SecuritySchemes if securitySchemes != nil { - log.Printf("%#v", securitySchemes) + //log.Printf("%#v", securitySchemes) api.Authentication = Authentication{ Required: true, @@ -386,7 +386,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri api.Authentication.Parameters[0].Name = securitySchemes["BearerAuth"].Value.Name api.Authentication.Parameters[0].In = securitySchemes["BearerAuth"].Value.In api.Authentication.Parameters[0].Scheme = securitySchemes["BearerAuth"].Value.Scheme - log.Printf("HANDLE BEARER AUTH") + //log.Printf("HANDLE BEARER AUTH") extraParameters = append(extraParameters, WorkflowAppActionParameter{ Name: "apikey", Description: "The apikey to use", @@ -402,7 +402,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri api.Authentication.Parameters[0].Name = securitySchemes["ApiKeyAuth"].Value.Name api.Authentication.Parameters[0].In = securitySchemes["ApiKeyAuth"].Value.In api.Authentication.Parameters[0].Scheme = securitySchemes["ApiKeyAuth"].Value.Scheme - log.Printf("HANDLE APIKEY AUTH") + //log.Printf("HANDLE APIKEY AUTH") extraParameters = append(extraParameters, WorkflowAppActionParameter{ Name: "apikey", Description: "The apikey to use", @@ -789,11 +789,11 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalParameters := []WorkflowAppActionParameter{} if len(path.Get.Parameters) > 0 { for _, param := range path.Get.Parameters { - if param.Value.Schema.Ref == "" { + //log.Printf("TYPE: %#v", param.Value.Schema) + if param.Value.Schema == nil { continue } - log.Printf("TYPE: %#v", param.Value.Schema) curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 80449235..37f5135e 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5181,7 +5181,7 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { if err != nil { log.Printf("Yaml error: %s", err) } else { - log.Printf("Successfully parsed YAML!") + //log.Printf("Successfully parsed YAML!") } } else { isJson = true @@ -5193,8 +5193,8 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { } if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { - log.Println("Handling v3 API") - swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + //log.Println("Handling v3 API") + _, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) if err != nil { return ParsedOpenApi{}, err } @@ -5203,18 +5203,18 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { hasher.Write(body) idstring = hex.EncodeToString(hasher.Sum(nil)) - log.Printf("Swagger v3 validation success with ID %s!", idstring) - log.Printf("Paths: %d", len(swagger.Paths)) + //log.Printf("Swagger v3 validation success with ID %s!", idstring) + //log.Printf("Paths: %d", len(swagger.Paths)) if !isJson { - log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) + //log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) } //return nil //return ParsedOpenApi{}, err } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { // Convert - log.Println("Handling v2 API") + //log.Println("Handling v2 API") var swagger openapi2.Swagger //log.Println(string(body)) err = json.Unmarshal(body, &swagger) @@ -5225,7 +5225,7 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { log.Printf("Yaml error: %s", err) return ParsedOpenApi{}, err } else { - log.Printf("Valid yaml!") + //log.Printf("Valid yaml!") } } @@ -5246,9 +5246,9 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { hasher.Write(swaggerdata) idstring = hex.EncodeToString(hasher.Sum(nil)) if !isJson { - log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring) + //log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring) } - log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring) + //log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring) ctx := context.Background() err = setOpenApiDatastore(ctx, idstring, parsed) @@ -5544,7 +5544,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Should generate yaml") + //log.Printf("Should generate yaml") api, pythonfunctions, err := generateYaml(swagger, newmd5) if err != nil { log.Printf("Failed building and generating yaml: %s", err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 6e0de95a..b62ce5dd 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2959,7 +2959,6 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D - log.Printf("Body: %s", string(body)) ctx := context.Background() workflowapps, err := getAllWorkflowApps(ctx) if err != nil { @@ -2978,7 +2977,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { appName := strings.ToLower(app.Name) appDesc := strings.ToLower(app.Description) if strings.Contains(appName, search) || strings.Contains(appDesc, search) { - log.Printf("Name: %s, Generated: %s, Activated: %s", app.Name, strconv.FormatBool(app.Generated), strconv.FormatBool(app.Activated)) + //log.Printf("Name: %s, Generated: %s, Activated: %s", app.Name, strconv.FormatBool(app.Generated), strconv.FormatBool(app.Activated)) returnValues = append(returnValues, app) } } @@ -2992,7 +2991,6 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { } returnData := fmt.Sprintf(`{"success": true, "reason": %s}`, string(newbody)) - log.Printf("Return: %s", returnData) resp.WriteHeader(200) resp.Write([]byte(returnData)) } @@ -3293,6 +3291,13 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { } func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { + + ctx := context.Background() + workflowapps, err := getAllWorkflowApps(ctx) + appCounter := 0 + if err != nil { + log.Printf("Failed to get existing generated apps") + } for _, file := range dir { if len(onlyname) > 0 && file.Name() != onlyname { continue @@ -3317,7 +3322,8 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, // Check the file filename := file.Name() if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") { - log.Printf("File: %s", filename) + appCounter += 1 + //log.Printf("File: %s", filename) //log.Printf("Found file: %s", filename) tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) @@ -3347,7 +3353,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, //log.Printf("%s", string(readFile)) swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body)) if err != nil { - log.Printf("Swagger validation error in loop: %s", err) + log.Printf("Swagger validation error in loop (%s): %s", filename, err) continue } @@ -3355,10 +3361,10 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, strings.Replace(swagger.Info.Title, " ", "", -1) } - log.Printf("Should generate yaml") + //log.Printf("Should generate yaml") api, _, err := generateYaml(swagger, parsedOpenApi.ID) if err != nil { - log.Printf("Failed building and generating yaml in loop: %s", err) + log.Printf("Failed building and generating yaml in loop (%s): %s", filename, err) continue } @@ -3369,20 +3375,38 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, api.Generated = true api.Activated = false - ctx := context.Background() - err = setWorkflowAppDatastore(ctx, api, api.ID) - if err != nil { - log.Printf("Failed setting workflowapp in loop: %s", err) - continue - } else { - log.Printf("Added %s:%s to the database from OpenAPI repo", api.Name, api.AppVersion) + found := false + for _, app := range workflowapps { + if app.ID == api.ID { + found = true + break + } else if app.Name == api.Name && app.AppVersion == api.AppVersion { + found = true + break + } } - return nil + if !found { + err = setWorkflowAppDatastore(ctx, api, api.ID) + if err != nil { + log.Printf("Failed setting workflowapp in loop: %s", err) + continue + } else { + log.Printf("Added %s:%s to the database from OpenAPI repo", api.Name, api.AppVersion) + } + } else { + //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) + } + + //return nil } } } + if appCounter > 0 { + log.Printf("Preloaded %d OpenApi apps in %s!", appCounter, extra) + } + return nil } diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index b4bfb87c..bf5adb80 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -186,7 +186,7 @@ const Apps = (props) => { } var imageline = data.large_image.length === 0 ? - Image missing + {data.title} : {data.title} @@ -356,7 +356,7 @@ const Apps = (props) => { - {selectedAction.parameters !== undefined ? + {selectedAction.parameters !== undefined && selectedAction.parameters !== null ?
Arguments {selectedAction.parameters.map(data => { From bd375f3a285a29939a221465a3379a387fc0592f Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 24 May 2020 06:09:52 +0200 Subject: [PATCH 05/13] Added datastore export to README --- README.md | 5 +++-- backend/go-app/main.go | 4 ++++ frontend/src/App.js | 4 ++-- frontend/src/Apps.js | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9353cc96..8f1b8a4b 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,16 @@ Documentation can be found on https://shuffler.io/docs/about or in your own inst Open an issue on Github, or [join the gitter chat](https://gitter.im/Shuffle-SOAR/community). For other / private requests: [frikky@shuffler.io](mailto:frikky@shuffler.io) ### Setup - Local development -Frontend - requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend - edit frontend/src/App.yaml (line 46~) from window.location.origin to http://YOUR IP:5001 +Frontend - requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend - edit frontend/src/App.yaml (line 44~) from window.location.origin to http://YOUR IP:5001 ```bash cd frontend npm i npm start ``` -Backend - API calls - requires [>=go1.13](https://golang.org/dl/) and [gcloud](https://cloud.google.com/sdk/install) +Backend - API calls - requires [>=go1.13](https://golang.org/dl/) ```bash +export DATASTORE_EMULATOR_HOST=0.0.0.0:8000 cd backend/go-app go build go run *.go diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 37f5135e..ef466fcc 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5824,6 +5824,9 @@ func runInit(ctx context.Context) { log.Printf("Downloading OpenAPI data for search - EXTRA APPS") apis := "https://github.com/frikky/OpenAPI-security-definitions" + + // THis gets memory problems hahah + //apis := "https://github.com/APIs-guru/openapi-directory" fs := memfs.New() storer := memory.NewStorage() cloneOptions := &git.CloneOptions{ @@ -5833,6 +5836,7 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed loading repo %s into memory: %s", err) } else { + log.Printf("Finished getting data.. Now looking for updates") dir, err := fs.ReadDir("") if err != nil { log.Printf("Failed reading folder: %s", err) diff --git a/frontend/src/App.js b/frontend/src/App.js index 63c4f860..0b65efd2 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -38,10 +38,10 @@ import AlertTemplate from "react-alert-template-basic"; import { positions, Provider } from "react-alert"; // Testing - localhost -const globalUrl = "http://192.168.3.6:5001" +//const globalUrl = "http://192.168.3.6:5001" // Production - backend proxy forwarding in nginx -//const globalUrl = window.location.origin +const globalUrl = window.location.origin const surfaceColor = "#27292D" const inputColor = "#383B40" diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index bf5adb80..dcc55e17 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -511,7 +511,7 @@ const Apps = (props) => { :

- Try a broader search term. E.g. "http" or "TheHive" + Try a broader search term, e.g. "http", "alert", "ticket" etc.

@@ -651,7 +651,7 @@ const Apps = (props) => { return response.json() }) .then((responseJson) => { - console.log(responseJson) + //console.log(responseJson) if (responseJson.success) { if (responseJson.reason !== undefined && responseJson.reason.length > 0) { setFilteredApps(responseJson.reason) From f94ef2bffa9bc441fad524c72bb61e258bae8f21 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 24 May 2020 06:50:59 +0200 Subject: [PATCH 06/13] Re-added app_sdk to main repo with MIT --- backend/Dockerfile | 13 +++++++++++-- backend/app_sdk/Dockerfile | 16 ++++++++++++++++ {functions/onprem => backend}/app_sdk/LICENSE | 0 backend/app_sdk/README.md | 16 ++++++++++++++++ backend/app_sdk/__init__.py | 0 .../onprem => backend}/app_sdk/app_base.py | 0 backend/app_sdk/build.sh | 4 ++++ backend/app_sdk/readme | 8 ++++++++ backend/app_sdk/requirements.txt | 2 ++ .../app_sdk}/static_baseline.py | 0 backend/go-app/codegen.go | 4 ++-- functions/README.md | 3 +++ functions/onprem/app_sdk/readme | 4 ---- 13 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 backend/app_sdk/Dockerfile rename {functions/onprem => backend}/app_sdk/LICENSE (100%) create mode 100644 backend/app_sdk/README.md create mode 100644 backend/app_sdk/__init__.py rename {functions/onprem => backend}/app_sdk/app_base.py (100%) create mode 100644 backend/app_sdk/build.sh create mode 100644 backend/app_sdk/readme create mode 100644 backend/app_sdk/requirements.txt rename {functions => backend/app_sdk}/static_baseline.py (100%) delete mode 100644 functions/onprem/app_sdk/readme diff --git a/backend/Dockerfile b/backend/Dockerfile index 36e3c0f2..a7a0de05 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,6 +2,7 @@ from golang as builder # Add files RUN mkdir /app +RUN mkdir /app_sdk WORKDIR /app ADD ./go-app/main.go /app ADD ./go-app/walkoff.go /app @@ -14,13 +15,21 @@ RUN go get -v RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . -# Certificate build +# Certificate build - gets required certs FROM alpine:latest as certs RUN apk --update add ca-certificates from scratch -COPY --from=builder /app/ / +RUN mkdir /app +RUN mkdir /app_sdk + +# Required files for code generation +ADD ./app_sdk/app_sdk.py /app_sdk +ADD ./app_sdk/static_baseline.py /app_sdk + +COPY --from=builder /app/ /app COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +WORKDIR /app EXPOSE 5001 CMD ["./webapp"] diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile new file mode 100644 index 00000000..44294a5b --- /dev/null +++ b/backend/app_sdk/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.7-alpine as base + +FROM base as builder +RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev + +RUN mkdir /install +WORKDIR /install + +COPY requirements.txt /requirements.txt +RUN pip install --prefix="/install" -r /requirements.txt + +FROM base + +COPY --from=builder /install /usr/local +COPY __init__.py /app/walkoff_app_sdk/__init__.py +COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/functions/onprem/app_sdk/LICENSE b/backend/app_sdk/LICENSE similarity index 100% rename from functions/onprem/app_sdk/LICENSE rename to backend/app_sdk/LICENSE diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md new file mode 100644 index 00000000..754392ff --- /dev/null +++ b/backend/app_sdk/README.md @@ -0,0 +1,16 @@ +# app_sdk.py +This is the SDK used for apps to behave like they should. +To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. + +# static_baseline.py +It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend. + +## If you want to update apps.. PS: downloads from docker hub do overrides.. :) +1. Write your code & check if runtime works +2. Build app_base image +3. docker rm $(docker ps -aq) # Remove all stopped containers +4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) +5. Rebuild the Docker image (click load in GUI?) + +# LICENSE +Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk/__init__.py b/backend/app_sdk/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/functions/onprem/app_sdk/app_base.py b/backend/app_sdk/app_base.py similarity index 100% rename from functions/onprem/app_sdk/app_base.py rename to backend/app_sdk/app_base.py diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh new file mode 100644 index 00000000..c93c83fc --- /dev/null +++ b/backend/app_sdk/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +docker rmi frikky/shuffle:app_sdk +docker build . -t frikky/shuffle:app_sdk --no-cache +docker push frikky/shuffle:app_sdk diff --git a/backend/app_sdk/readme b/backend/app_sdk/readme new file mode 100644 index 00000000..4a01094d --- /dev/null +++ b/backend/app_sdk/readme @@ -0,0 +1,8 @@ +# App_sdk +App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk + +# Added static:baseline.py +It's used for python code generation and should be under MIT. + +# LICENSE +Everything in this folder is MIT, not AGPLv3 diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt new file mode 100644 index 00000000..804abb1b --- /dev/null +++ b/backend/app_sdk/requirements.txt @@ -0,0 +1,2 @@ +requests +urllib3 diff --git a/functions/static_baseline.py b/backend/app_sdk/static_baseline.py similarity index 100% rename from functions/static_baseline.py rename to backend/app_sdk/static_baseline.py diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 889c20e7..29c06917 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -122,8 +122,8 @@ func streamZipdata(ctx context.Context, identifier, pythoncode, requirements str func getAppbase() ([]byte, []byte, error) { // 1. Have baseline in bucket/generated_apps/baseline // 2. Copy the baseline to a new folder with identifier name - static := "../../functions/static_baseline.py" - appbase := "../../functions/onprem/app_sdk/app_base.py" + static := "../app_sdk/static_baseline.py" + appbase := "../app_sdk/app_base.py" staticData, err := ioutil.ReadFile(static) if err != nil { diff --git a/functions/README.md b/functions/README.md index 76682e26..b953b6c0 100644 --- a/functions/README.md +++ b/functions/README.md @@ -17,3 +17,6 @@ The point of this folder is to make GCP Cloud functions able to run default WALK * class AppBase * class * Runner + +## Update may 2020: +Moved static_baseline to ./onprem/app_sdk because of license. diff --git a/functions/onprem/app_sdk/readme b/functions/onprem/app_sdk/readme deleted file mode 100644 index 873d45d3..00000000 --- a/functions/onprem/app_sdk/readme +++ /dev/null @@ -1,4 +0,0 @@ -# App_sdk -App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk - -It's under MIT license, NOT AGPLv3. From 62bc6708c1d04ca4c0aea5e149013207c243b511 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 24 May 2020 07:53:39 +0200 Subject: [PATCH 07/13] Rebuild Shuffle structure for docker image builds --- .gitignore | 3 + backend/Dockerfile | 13 +- backend/README.md | 69 +---- backend/app_gen/LICENSE | 21 ++ .../generated/carbon_black_response.yaml | 0 .../generated/cyberreason.yaml | 0 .../generated/recorded_future.yaml | 0 .../openapi-parsers/generated/shodan.yaml | 0 .../generated/tenable_tenable.io.yaml | 0 .../app_gen}/openapi-parsers/misp.py | 0 .../other/TIO-API-Container-Security-v1.json | 0 .../other/TIO-API-Container-Security-v2.json | 0 .../other/TIO-API-Downloads-API.json | 0 .../other/TIO-API-Tenable-Platform.json | 0 .../TIO-API-Vulnerability-Management.json | 0 .../TIO-API-Web-Application-Scanning.json | 0 .../app_gen}/openapi-parsers/swimlane.py | 0 .../app_gen}/openapi/README.md | 0 .../app_gen}/openapi/baseline/Dockerfile | 0 .../openapi/baseline/requirements.txt | 0 {app_gen => backend/app_gen}/openapi/test.go | 0 .../app_gen}/openapi/testGCP.go | 0 .../app_gen}/python-lib/README.md | 0 .../app_gen}/python-lib/baseline/Dockerfile | 0 .../python-lib/baseline/docker-compose.yml | 0 .../app_gen}/python-lib/baseline/env.txt | 0 .../python-lib/baseline/requirements.txt | 0 .../app_gen}/python-lib/generator.py | 0 .../app_gen}/python-lib/requirements.txt | 0 backend/app_sdk/readme | 8 - backend/build.sh | 13 + backend/deploy-backend.sh | 8 - backend/go-app/codegen.go | 20 +- backend/go-app/main.go | 2 +- backend/run.sh | 15 - backend/webhook/Dockerfile | 15 - backend/webhook/README.md | 7 - backend/webhook/functionhook.go | 44 --- backend/webhook/gcp_run.sh | 4 - backend/webhook/run.sh | 16 -- backend/webhook/webhook.go | 260 ------------------ frontend/src/App.js | 6 +- frontend/src/Apps.js | 21 +- 43 files changed, 97 insertions(+), 448 deletions(-) create mode 100644 backend/app_gen/LICENSE rename {app_gen => backend/app_gen}/openapi-parsers/generated/carbon_black_response.yaml (100%) rename {app_gen => backend/app_gen}/openapi-parsers/generated/cyberreason.yaml (100%) rename {app_gen => backend/app_gen}/openapi-parsers/generated/recorded_future.yaml (100%) rename {app_gen => backend/app_gen}/openapi-parsers/generated/shodan.yaml (100%) rename {app_gen => backend/app_gen}/openapi-parsers/generated/tenable_tenable.io.yaml (100%) rename {app_gen => backend/app_gen}/openapi-parsers/misp.py (100%) rename {app_gen => backend/app_gen}/openapi-parsers/other/TIO-API-Container-Security-v1.json (100%) rename {app_gen => backend/app_gen}/openapi-parsers/other/TIO-API-Container-Security-v2.json (100%) rename {app_gen => backend/app_gen}/openapi-parsers/other/TIO-API-Downloads-API.json (100%) rename {app_gen => backend/app_gen}/openapi-parsers/other/TIO-API-Tenable-Platform.json (100%) rename {app_gen => backend/app_gen}/openapi-parsers/other/TIO-API-Vulnerability-Management.json (100%) rename {app_gen => backend/app_gen}/openapi-parsers/other/TIO-API-Web-Application-Scanning.json (100%) rename {app_gen => backend/app_gen}/openapi-parsers/swimlane.py (100%) rename {app_gen => backend/app_gen}/openapi/README.md (100%) rename {app_gen => backend/app_gen}/openapi/baseline/Dockerfile (100%) rename {app_gen => backend/app_gen}/openapi/baseline/requirements.txt (100%) rename {app_gen => backend/app_gen}/openapi/test.go (100%) rename {app_gen => backend/app_gen}/openapi/testGCP.go (100%) rename {app_gen => backend/app_gen}/python-lib/README.md (100%) rename {app_gen => backend/app_gen}/python-lib/baseline/Dockerfile (100%) rename {app_gen => backend/app_gen}/python-lib/baseline/docker-compose.yml (100%) rename {app_gen => backend/app_gen}/python-lib/baseline/env.txt (100%) rename {app_gen => backend/app_gen}/python-lib/baseline/requirements.txt (100%) rename {app_gen => backend/app_gen}/python-lib/generator.py (100%) rename {app_gen => backend/app_gen}/python-lib/requirements.txt (100%) delete mode 100644 backend/app_sdk/readme create mode 100644 backend/build.sh delete mode 100644 backend/deploy-backend.sh delete mode 100644 backend/run.sh delete mode 100644 backend/webhook/Dockerfile delete mode 100644 backend/webhook/README.md delete mode 100644 backend/webhook/functionhook.go delete mode 100644 backend/webhook/gcp_run.sh delete mode 100644 backend/webhook/run.sh delete mode 100644 backend/webhook/webhook.go diff --git a/.gitignore b/.gitignore index 0aa56849..5bebe49f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ backend/go-app/generated* functions/generated_apps *.zip +*openapi-parsers/generated/* +*openapi-parsers/other/* + backend/onprem/app_sdk/apps *test.py diff --git a/backend/Dockerfile b/backend/Dockerfile index a7a0de05..14d9137c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -11,6 +11,11 @@ ADD ./go-app/codegen.go /app ADD ./go-app/go.mod /app +# Required files for code generation +ADD ./app_sdk/app_base.py /app_sdk +ADD ./app_sdk/static_baseline.py /app_sdk +ADD ./app_gen /app_gen + RUN go get -v RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . @@ -20,14 +25,10 @@ FROM alpine:latest as certs RUN apk --update add ca-certificates from scratch -RUN mkdir /app -RUN mkdir /app_sdk - -# Required files for code generation -ADD ./app_sdk/app_sdk.py /app_sdk -ADD ./app_sdk/static_baseline.py /app_sdk COPY --from=builder /app/ /app +COPY --from=builder /app_sdk/ /app_sdk +COPY --from=builder /app_gen/ /app_gen COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt WORKDIR /app diff --git a/backend/README.md b/backend/README.md index 8cea3c80..4a3e0599 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,61 +1,18 @@ -# Backend setup -1 Go to https://console.cloud.google.com/apis/credentials?project=shuffle-241517&folder&organizationId and get credentials -2. Move the file to current folder (or make step 3 be your download folder or w/e) -3. export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/Shuffle-2a19ff64af66.json +# Backend +This folder has all parts necessary for the backend to run locally and in Docker -# Backend run testserver (appengine) -1. Set up gcloud locally -```bash -dev_appserver.py go-app/ --port=5001 --host=0.0.0.0 --enable_host_checking=false -``` +## Structure +* go-app: The backend. Modify these to edit the backend API. +* database: The datastore database. +* app_sdk: The app_sdk for apps. MIT licensed. +* app_gen: Code used when generating docker images. MIT licensed +* tests: A bunch of cronscripts. There are no real, good tests yet -# Backend deploy -* I created a simple script that moves the data into your GOPATH and deploys for you. This will require more tests in the future. +## Development +Shuffle's backend is written in Go, with apps being python (for now). More about local development can be seen in the main README. -# OpenAPI spec checks -Paths: -* /path/{variablename}?queryvar= <-- variable -* ^variablename needs to be part of parameters too. -* ^queryvar needs to be part of parameters too. +Running the backend: ``` -parameters: -- name: variablename - in: path - description: Blah blah - required: true/false - schema: - type: string - enum: [a, b, c] # <-- not necessary, but could be great -- name: queryvar - in: query - description: blah blah - required: true/false - schema: - type: string - enum: [a, b, c] -``` -* requestBody? Not in GET, DELETE & HEAD. Can consume JSON, XML, form data, plai ntext & others. Can use markdown for the description. -* Do I care about the response? Maybe :o - -``` -requestBody: - description: Optional kind of description - required: false/true - content: - application/json: - schema: - type: object - additionalProperties: true - properties: - name: - type: string - fav_number: - type: integer - required: - - name - - email - encoding: - color: - style: form - explode: false +cd go-app +go run *.go ``` diff --git a/backend/app_gen/LICENSE b/backend/app_gen/LICENSE new file mode 100644 index 00000000..ce11f6f3 --- /dev/null +++ b/backend/app_gen/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Frikkylikeme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/app_gen/openapi-parsers/generated/carbon_black_response.yaml b/backend/app_gen/openapi-parsers/generated/carbon_black_response.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/carbon_black_response.yaml rename to backend/app_gen/openapi-parsers/generated/carbon_black_response.yaml diff --git a/app_gen/openapi-parsers/generated/cyberreason.yaml b/backend/app_gen/openapi-parsers/generated/cyberreason.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/cyberreason.yaml rename to backend/app_gen/openapi-parsers/generated/cyberreason.yaml diff --git a/app_gen/openapi-parsers/generated/recorded_future.yaml b/backend/app_gen/openapi-parsers/generated/recorded_future.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/recorded_future.yaml rename to backend/app_gen/openapi-parsers/generated/recorded_future.yaml diff --git a/app_gen/openapi-parsers/generated/shodan.yaml b/backend/app_gen/openapi-parsers/generated/shodan.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/shodan.yaml rename to backend/app_gen/openapi-parsers/generated/shodan.yaml diff --git a/app_gen/openapi-parsers/generated/tenable_tenable.io.yaml b/backend/app_gen/openapi-parsers/generated/tenable_tenable.io.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/tenable_tenable.io.yaml rename to backend/app_gen/openapi-parsers/generated/tenable_tenable.io.yaml diff --git a/app_gen/openapi-parsers/misp.py b/backend/app_gen/openapi-parsers/misp.py similarity index 100% rename from app_gen/openapi-parsers/misp.py rename to backend/app_gen/openapi-parsers/misp.py diff --git a/app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json b/backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json b/backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Downloads-API.json b/backend/app_gen/openapi-parsers/other/TIO-API-Downloads-API.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Downloads-API.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Downloads-API.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json b/backend/app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json b/backend/app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json b/backend/app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json diff --git a/app_gen/openapi-parsers/swimlane.py b/backend/app_gen/openapi-parsers/swimlane.py similarity index 100% rename from app_gen/openapi-parsers/swimlane.py rename to backend/app_gen/openapi-parsers/swimlane.py diff --git a/app_gen/openapi/README.md b/backend/app_gen/openapi/README.md similarity index 100% rename from app_gen/openapi/README.md rename to backend/app_gen/openapi/README.md diff --git a/app_gen/openapi/baseline/Dockerfile b/backend/app_gen/openapi/baseline/Dockerfile similarity index 100% rename from app_gen/openapi/baseline/Dockerfile rename to backend/app_gen/openapi/baseline/Dockerfile diff --git a/app_gen/openapi/baseline/requirements.txt b/backend/app_gen/openapi/baseline/requirements.txt similarity index 100% rename from app_gen/openapi/baseline/requirements.txt rename to backend/app_gen/openapi/baseline/requirements.txt diff --git a/app_gen/openapi/test.go b/backend/app_gen/openapi/test.go similarity index 100% rename from app_gen/openapi/test.go rename to backend/app_gen/openapi/test.go diff --git a/app_gen/openapi/testGCP.go b/backend/app_gen/openapi/testGCP.go similarity index 100% rename from app_gen/openapi/testGCP.go rename to backend/app_gen/openapi/testGCP.go diff --git a/app_gen/python-lib/README.md b/backend/app_gen/python-lib/README.md similarity index 100% rename from app_gen/python-lib/README.md rename to backend/app_gen/python-lib/README.md diff --git a/app_gen/python-lib/baseline/Dockerfile b/backend/app_gen/python-lib/baseline/Dockerfile similarity index 100% rename from app_gen/python-lib/baseline/Dockerfile rename to backend/app_gen/python-lib/baseline/Dockerfile diff --git a/app_gen/python-lib/baseline/docker-compose.yml b/backend/app_gen/python-lib/baseline/docker-compose.yml similarity index 100% rename from app_gen/python-lib/baseline/docker-compose.yml rename to backend/app_gen/python-lib/baseline/docker-compose.yml diff --git a/app_gen/python-lib/baseline/env.txt b/backend/app_gen/python-lib/baseline/env.txt similarity index 100% rename from app_gen/python-lib/baseline/env.txt rename to backend/app_gen/python-lib/baseline/env.txt diff --git a/app_gen/python-lib/baseline/requirements.txt b/backend/app_gen/python-lib/baseline/requirements.txt similarity index 100% rename from app_gen/python-lib/baseline/requirements.txt rename to backend/app_gen/python-lib/baseline/requirements.txt diff --git a/app_gen/python-lib/generator.py b/backend/app_gen/python-lib/generator.py similarity index 100% rename from app_gen/python-lib/generator.py rename to backend/app_gen/python-lib/generator.py diff --git a/app_gen/python-lib/requirements.txt b/backend/app_gen/python-lib/requirements.txt similarity index 100% rename from app_gen/python-lib/requirements.txt rename to backend/app_gen/python-lib/requirements.txt diff --git a/backend/app_sdk/readme b/backend/app_sdk/readme deleted file mode 100644 index 4a01094d..00000000 --- a/backend/app_sdk/readme +++ /dev/null @@ -1,8 +0,0 @@ -# App_sdk -App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk - -# Added static:baseline.py -It's used for python code generation and should be under MIT. - -# LICENSE -Everything in this folder is MIT, not AGPLv3 diff --git a/backend/build.sh b/backend/build.sh new file mode 100644 index 00000000..5679ae8b --- /dev/null +++ b/backend/build.sh @@ -0,0 +1,13 @@ +#!/bin/sh +docker stop shuffle-backend +docker rm shuffle-backend +docker rmi frikky/shuffle:backend + +docker build . -t frikky/shuffle:backend + +echo "Starting server" +#docker run -it \ +# -p 5001:5001 \ +# -v /var/run/docker.sock:/var/run/docker.sock \ +# --env DATASTORE_EMULATOR_HOST=192.168.3.6:8000 \ +# frikky/shuffle:backend diff --git a/backend/deploy-backend.sh b/backend/deploy-backend.sh deleted file mode 100644 index bb51e6de..00000000 --- a/backend/deploy-backend.sh +++ /dev/null @@ -1,8 +0,0 @@ -# Deploys to backend -echo "Deploying to appengine." -mkdir -p $GOPATH/src/github.com/frikky/shuffle -cp -r go-app/* $GOPATH/src/github.com/frikky/shuffle -cd $GOPATH/src/github.com/frikky/shuffle -go build -go test -gcloud app deploy $GOPATH/src/github.com/frikky/shuffle/app.yaml diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 29c06917..d37954a9 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -218,7 +218,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { // adding md5 based on input data to not overwrite earlier data. generatedPath := "generated" - subpath := "../../app_gen/openapi/" + subpath := "../app_gen/openapi/" identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) appPath := fmt.Sprintf("%s/%s", generatedPath, identifier) @@ -704,6 +704,9 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ optionalParameters := []WorkflowAppActionParameter{} if len(path.Connect.Parameters) > 0 { for _, param := range path.Connect.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -876,6 +879,9 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalParameters := []WorkflowAppActionParameter{} if len(path.Head.Parameters) > 0 { for _, param := range path.Head.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -958,6 +964,9 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] optionalParameters := []WorkflowAppActionParameter{} if len(path.Delete.Parameters) > 0 { for _, param := range path.Delete.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1052,6 +1061,9 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } if len(path.Post.Parameters) > 0 { for _, param := range path.Post.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1145,6 +1157,9 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W } if len(path.Patch.Parameters) > 0 { for _, param := range path.Patch.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1238,6 +1253,9 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } if len(path.Put.Parameters) > 0 { for _, param := range path.Put.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, diff --git a/backend/go-app/main.go b/backend/go-app/main.go index ef466fcc..54cd69ba 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5698,7 +5698,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { log.Println(len(user.PrivateApps)) c, err := request.Cookie("session_token") if err == nil { - log.Printf("Should've deleted cache for %s with token %s", user.Username, c.Value) + //log.Printf("Should've deleted cache for %s with token %s", user.Username, c.Value) //err = memcache.Delete(request.Context(), c.Value) //err = memcache.Delete(request.Context(), user.ApiKey) } diff --git a/backend/run.sh b/backend/run.sh deleted file mode 100644 index edfe195c..00000000 --- a/backend/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# docker stop nginx -# docker rm nginx -# docker rmi nginx -# -# echo "Running build for website" -# sudo npm run build -# docker build . -t nginx - -echo "Starting server" -docker run -it \ - -p 5001:5001 \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --env DATASTORE_EMULATOR_HOST=192.168.3.6:8000 \ - frikky/shuffle:backend diff --git a/backend/webhook/Dockerfile b/backend/webhook/Dockerfile deleted file mode 100644 index 4b709eb5..00000000 --- a/backend/webhook/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Build environment -# production environment -from golang as builder - -RUN go get github.com/gorilla/handlers -RUN go get github.com/gorilla/mux - -WORKDIR /app -COPY webhook.go /app/webhook.go -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webhook . - -from scratch -COPY --from=builder /app/ / - -CMD ["./webhook"] diff --git a/backend/webhook/README.md b/backend/webhook/README.md deleted file mode 100644 index 8b87bffb..00000000 --- a/backend/webhook/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Steps to deploy to Google cloud function -1. -```bash -zip webhook.zip * -``` -2. Go to google cloud bucket and upload the zip -3. Go to worker for webhook and upload diff --git a/backend/webhook/functionhook.go b/backend/webhook/functionhook.go deleted file mode 100644 index 2df691b0..00000000 --- a/backend/webhook/functionhook.go +++ /dev/null @@ -1,44 +0,0 @@ -package function - -import ( - "encoding/json" - "io/ioutil" - "log" - "net/http" - "time" -) - -// GetUserDetails - Get one user's details from randomuser.me API -func GetUserDetails(w http.ResponseWriter, r *http.Request) { - randomUserClient := http.Client{ - Timeout: time.Second * 3, - } - - req, err := http.NewRequest(http.MethodGet, "https://randomuser.me/api/", nil) - if err != nil { - log.Fatal(err) - return - } - - res, err2 := randomUserClient.Do(req) - if err2 != nil { - log.Fatal(err2) - return - } - - body, err3 := ioutil.ReadAll(res.Body) - if err3 != nil { - log.Fatal(err3) - } - - var o map[string]interface{} - json.Unmarshal([]byte(body), &o) - - results := o["results"].([]interface{}) - result := results[0].(map[string]interface{}) - - result["generator"] = "google-cloud-function" - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(result) -} diff --git a/backend/webhook/gcp_run.sh b/backend/webhook/gcp_run.sh deleted file mode 100644 index 605777b5..00000000 --- a/backend/webhook/gcp_run.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker build . -t gcr.io/shuffle-241517/webhook -docker push gcr.io/shuffle-241517/webhook - -gcloud beta run deploy webhook --image gcr.io/shuffle-241517/webhook diff --git a/backend/webhook/run.sh b/backend/webhook/run.sh deleted file mode 100644 index c9914de7..00000000 --- a/backend/webhook/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -docker stop webhook -docker rm webhook -docker rmi webhook - -docker build . -t webhook -docker run -d \ - -e "HOOKPORT=5001" \ - -e "URIPATH=/webhook" \ - -e "CALLBACKURL=http://192.168.3.6:5001/api/v1/hooks/d6ef8912e8bd37776e654cbc14c2629c/result" \ - -p 6000:6000 \ - --name webhook \ - -h webhook \ - --restart always \ - webhook - -docker logs -f webhook diff --git a/backend/webhook/webhook.go b/backend/webhook/webhook.go deleted file mode 100644 index bd256d78..00000000 --- a/backend/webhook/webhook.go +++ /dev/null @@ -1,260 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - - "github.com/gorilla/handlers" - "github.com/gorilla/mux" -) - -type Info struct { - Url string `json:"url" datastore:"url"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` -} - -// Actions to be done by webhooks etc -// Field is the actual field to use from json -type HookAction struct { - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Id string `json:"id" datastore:"id"` - Field string `json:"field" datastore:"field"` -} - -type Hook struct { - Id string `json:"id" datastore:"id"` - Info Info `json:"info" datastore:"info"` - Transforms struct{} `json:"transforms" datastore:"transforms"` - Actions []HookAction `json:"actions" datastore:"actions"` - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Running bool `json:"running" datastore:"running"` -} - -var hook Hook - -func handleWorkflowAction(request *http.Request, action HookAction) error { - //log.Printf("WORKFLOW!: %#v", action) - log.Printf("Should execute workflow %s", action.Id) - - callbackUrl := os.Getenv("CALLBACKURL") - apikey := os.Getenv("APIKEY") - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, action.Id) - - // ret = requests.post(fullurl, headers=headers, json=data) - //if ret.status_code != 202: - // print(ret.text) - // print(ret.status_code) - // print("Exiting workflows - run queue") - // exit() - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - return err - } - - // Execute a workflow - client := &http.Client{} - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer(body), - ) - - if err != nil { - log.Printf("Error making http request: %s", req) - return err - } - - req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey)) - req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - log.Printf("Error in http request: %s", req) - } - - log.Printf("%#v", resp) - return nil -} - -// FIXME - refresh hook information once in a while. Compare timestamps or something -func callback(resp http.ResponseWriter, request *http.Request) { - //apikey = os.Getenv("APIKEY") - //hookId = os.Getenv("HOOKID") - - handledWorkflowIds := []string{} - for _, item := range hook.Actions { - if item.Type == "" { - log.Printf("CONTINUE AAS EMPTY ITEM: %#v", item) - continue - } - - if item.Type == "workflow" { - found := false - for _, workflowId := range handledWorkflowIds { - if item.Id == workflowId { - found = true - break - } - } - - if found { - continue - } - - handledWorkflowIds = append(handledWorkflowIds, item.Id) - err := handleWorkflowAction(request, item) - if err != nil { - log.Printf("Error in workflow exec: %s", err) - } - } - } - - // FIXME - send the webhookdata to a logging service? Idk - //body, err := ioutil.ReadAll(request.Body) - //if err != nil { - // log.Println("Failed reading body") - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - // return - //} - - //callback, err := http.Post(callbackUrl, "application/json", bytes.NewBuffer(body)) - //if err != nil { - // log.Printf("Failed sending callback to %s", callbackUrl) - //} - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return -} - -func loadConfiguration(fullUrl string, apikey string) error { - client := &http.Client{} - - req, err := http.NewRequest( - "GET", - fullUrl, - nil, - ) - - if err != nil { - log.Printf("Error making http request: %s", req) - return err - } - - req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey)) - req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - log.Printf("Error in http request: %s", req) - return err - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Printf("Error reading response: %s", req) - return err - } - - err = json.Unmarshal(body, &hook) - if err != nil { - log.Printf("Failed unmarshaling hook API", req) - return err - } - - log.Printf("%#v", hook) - log.Println(hook.Actions) - return nil -} - -func webhook() { - // FIXME - remove static - ip := "0.0.0.0" - - // Basic webserver stuff - baseFilePath := os.Getenv("URIPATH") - basePort := os.Getenv("HOOKPORT") - callbackUrl := os.Getenv("CALLBACKURL") - apikey := os.Getenv("APIKEY") - hookId := os.Getenv("HOOKID") - - if len(baseFilePath) == 0 { - log.Println("Env URIPATH not set") - os.Exit(3) - } - - if len(basePort) == 0 { - log.Println("Env HOOKPORT not set") - os.Exit(3) - } - - if len(callbackUrl) == 0 { - log.Println("Env CALLBACKURL not set") - os.Exit(3) - } - - if len(apikey) == 0 { - log.Println("Env APIKEY not set") - os.Exit(3) - } - - if len(hookId) == 0 { - log.Println("Env HOOKID not set") - os.Exit(3) - } - - log.Println("Loading hook configuration") - err := loadConfiguration( - fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId), - apikey, - ) - - if err != nil { - log.Fatalf("Error loading config: %s", err) - } - - // Optional - // if len(callbackOpts) == 0 { - // log.Println("Env CALLBACKOPTS not set") - // os.Exit(3) - // } - - port := fmt.Sprintf(":%s", basePort) - log.Printf("Starting webhook on %s%s with path %s", ip, port, baseFilePath) - - // Routing - mux := mux.NewRouter() - mux.SkipClean(true) - - // FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend - mux.HandleFunc(baseFilePath, callback).Methods("POST") - - handlers.LoggingHandler(os.Stdout, mux) - loggedRouter := handlers.LoggingHandler(os.Stdout, mux) - - err = http.ListenAndServe( - port, - loggedRouter, - ) - - if err != nil { - log.Fatal("ListenAndServer: ", err) - } -} - -func F(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Write([]byte(r.Header.Get("X-Forwarded-For"))) -} - -func main() { - webhook() -} diff --git a/frontend/src/App.js b/frontend/src/App.js index 0b65efd2..f22d9f9f 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -38,10 +38,10 @@ import AlertTemplate from "react-alert-template-basic"; import { positions, Provider } from "react-alert"; // Testing - localhost -//const globalUrl = "http://192.168.3.6:5001" +const globalUrl = "http://192.168.3.6:5001" // Production - backend proxy forwarding in nginx -const globalUrl = window.location.origin +//const globalUrl = window.location.origin const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -77,7 +77,7 @@ const App = (message, props) => { }}) if (isLoaded && !isLoggedIn && (!window.location.pathname.startsWith("/login") && (!window.location.pathname.startsWith("/docs") && (!window.location.pathname.startsWith("/adminsetup"))))) { - window.location = "login" + window.location = "/login" } const checkLogin = () => { diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index dcc55e17..e5c3b5bf 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -286,7 +286,7 @@ const Apps = (props) => { var description = selectedApp.description const url = "/apps/edit/"+selectedApp.id - var editButton = selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ? + var editButton = selectedApp.activated && selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ? : null + var activateButton = selectedApp.generated && !selectedApp.activated ? + + : null + var deleteButton = (selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true) ?
: null} + {activateButton} {editButton} {deleteButton}
@@ -384,11 +396,12 @@ const Apps = (props) => { null return( -
+

App Creator

- Security API's + How it works +  - Security API's  - OpenAPI directory
Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available. @@ -445,7 +458,7 @@ const Apps = (props) => {

Upload

- +
From eb59c0cea6164cc7f3b6c6651b091c7163d68461 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 24 May 2020 08:57:26 +0200 Subject: [PATCH 08/13] Finished app activation with search engine --- backend/go-app/codegen.go | 1 + backend/go-app/main.go | 50 +++++++++++++------------------------- backend/go-app/walkoff.go | 18 ++++++++++++-- frontend/src/AppCreator.js | 32 ++++++++++++++++-------- frontend/src/Apps.js | 44 ++++++++++++++++++++------------- 5 files changed, 83 insertions(+), 62 deletions(-) diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index d37954a9..abe03fa8 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -361,6 +361,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri api.Tested = false api.PrivateID = newmd5 api.Generated = true + api.Activated = true // Setting up security schemes extraParameters := []WorkflowAppActionParameter{} diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 54cd69ba..ba9a5477 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5194,24 +5194,21 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { //log.Println("Handling v3 API") - _, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + swaggerv3, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) if err != nil { return ParsedOpenApi{}, err } - hasher := md5.New() - hasher.Write(body) - idstring = hex.EncodeToString(hasher.Sum(nil)) - - //log.Printf("Swagger v3 validation success with ID %s!", idstring) - //log.Printf("Paths: %d", len(swagger.Paths)) - - if !isJson { - //log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) + swaggerdata, err = json.Marshal(swaggerv3) + if err != nil { + log.Printf("Failed unmarshaling v3 data: %s", err) + return ParsedOpenApi{}, err } - //return nil - //return ParsedOpenApi{}, err + hasher := md5.New() + hasher.Write(swaggerdata) + idstring = hex.EncodeToString(hasher.Sum(nil)) + } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { // Convert //log.Println("Handling v2 API") @@ -5245,24 +5242,19 @@ func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { hasher := md5.New() hasher.Write(swaggerdata) idstring = hex.EncodeToString(hasher.Sum(nil)) - if !isJson { - //log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring) - } - //log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring) - - ctx := context.Background() - err = setOpenApiDatastore(ctx, idstring, parsed) - if err != nil { - log.Printf("Failed uploading openapi2 to datastore: %s", err) - return ParsedOpenApi{}, err - } - } if len(swaggerdata) > 0 { body = swaggerdata } + //parsed := ParsedOpenApi{} + if !isJson { + //yaml.l + log.Printf("Parsing from YAML to json!") + body = swaggerdata + } + // Parsing it to swagger 3 parsed = ParsedOpenApi{ ID: idstring, @@ -5695,14 +5687,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Println(len(user.PrivateApps)) - c, err := request.Cookie("session_token") - if err == nil { - //log.Printf("Should've deleted cache for %s with token %s", user.Username, c.Value) - //err = memcache.Delete(request.Context(), c.Value) - //err = memcache.Delete(request.Context(), user.ApiKey) - } - parsed := ParsedOpenApi{ ID: api.ID, Body: string(body), @@ -5836,7 +5820,7 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed loading repo %s into memory: %s", err) } else { - log.Printf("Finished getting data.. Now looking for updates") + log.Printf("Finished git clone. Looking for updates to the repo.") dir, err := fs.ReadDir("") if err != nil { log.Printf("Failed reading folder: %s", err) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b62ce5dd..67d4835d 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2704,7 +2704,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { } location := strings.Split(request.URL.String(), "/") - log.Printf("%#v", location) var fileId string if location[1] == "api" { if len(location) <= 4 { @@ -2732,14 +2731,22 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("Getting app %s", fileId) parsedApi, err := getOpenApiDatastore(ctx, fileId) if err != nil { + log.Printf("OpenApi doesn't exist for: %s - err: %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - parsedApi.Success = true + //log.Printf("Parsed API: %#v", parsedApi) + if len(parsedApi.ID) > 0 { + parsedApi.Success = true + } else { + parsedApi.Success = false + } + data, err := json.Marshal(parsedApi) if err != nil { resp.WriteHeader(422) @@ -3393,6 +3400,13 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, continue } else { log.Printf("Added %s:%s to the database from OpenAPI repo", api.Name, api.AppVersion) + + // Set OpenAPI datastore + err = setOpenApiDatastore(ctx, parsedOpenApi.ID, parsedOpenApi) + if err != nil { + log.Printf("Failed uploading openapi to datastore in loop: %s", err) + continue + } } } else { //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) diff --git a/frontend/src/AppCreator.js b/frontend/src/AppCreator.js index 5e85376b..24ab8353 100644 --- a/frontend/src/AppCreator.js +++ b/frontend/src/AppCreator.js @@ -81,7 +81,7 @@ const AppCreator = (props) => { const [actionsModalOpen, setActionsModalOpen] = useState(false); const [authenticationOption, setAuthenticationOption] = useState(authenticationOptions[0]); const [parameterName, setParameterName] = useState(""); - const [parameterLocation, setParameterLocation] = useState(apikeySelection[0]); + const [parameterLocation, setParameterLocation] = useState(apikeySelection.length > 0 ? apikeySelection[0] : ""); const [urlPath, setUrlPath] = useState(""); //const [urlPathQueries, setUrlPathQueries] = useState([{"name": "test", "required": false}]); const [urlPathQueries, setUrlPathQueries] = useState([]); @@ -161,7 +161,7 @@ const AppCreator = (props) => { .then((responseJson) => { setIsAppLoaded(true) if (!responseJson.success) { - alert.error("Failed to verify") + alert.error("Failed to get the app") } else { const data = JSON.parse(responseJson.body) console.log("LOADED IMAGE: ", data.image) @@ -263,9 +263,12 @@ const AppCreator = (props) => { break } else if (value.type === "apiKey") { setAuthenticationOption("API key") - setParameterName(value.name) + setParameterName(value.name) + + value.in = value.in.charAt(0).toUpperCase() + value.in.slice(1); setParameterLocation(value.in) if (!apikeySelection.includes(value.in)) { + console.log("APIKEY SELECT: ", apikeySelection) alert.error("Might be error in setting up API key authentication") } break @@ -593,6 +596,9 @@ const AppCreator = (props) => { setActions(actions) } + console.log("Option: ", authenticationOption) + console.log("Location: ", parameterLocation) + console.log("Name: ", parameterName) const apiKey = authenticationOption === "API key" ?

API key

@@ -605,6 +611,7 @@ const AppCreator = (props) => { id="standard-required" margin="normal" variant="outlined" + defaultValue={parameterName} helperText={
Can't be empty. Can't contain any of the following characters: !#$%&'^+-._~|]+$
} onChange={e => setParameterName(e.target.value)} InputProps={{ @@ -628,13 +635,18 @@ const AppCreator = (props) => { name: 'age', id: 'outlined-age-simple', }} - > - > - {apikeySelection.map(data => ( - - {data} - - ))} + > + {apikeySelection.map(data => { + if (data === undefined) { + return null + } + + return ( + + {data} + + )} + )}
diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index e5c3b5bf..8fdde37b 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -262,7 +262,7 @@ const Apps = (props) => { const dividerColor = "rgb(225, 228, 232)" const uploadViewPaperStyle = { minWidth: "100%", - maxWidth: "100%", + maxWidth: 662.5, color: "white", backgroundColor: surfaceColor, display: "flex", @@ -285,9 +285,10 @@ const Apps = (props) => { var description = selectedApp.description - const url = "/apps/edit/"+selectedApp.id + const editUrl = "/apps/edit/"+selectedApp.id + const activateUrl = "/apps/new?id="+selectedApp.id var editButton = selectedApp.activated && selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ? - + : null var activateButton = selectedApp.generated && !selectedApp.activated ? - + : null - - var deleteButton = (selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true) ? + var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ? : null + var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? + {selectedApp.title} + : + {selectedApp.title} //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), { var baseInfo = newAppname.length > 0 ?
-

{newAppname}

-

{description}

+
+
+ {imageline} +
+
+

{newAppname}

+

{description}

+
+
+ {activateButton} + {editButton} + {deleteButton} - {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} -

ID: {selectedApp.id}

- {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null} + {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} +

ID: {selectedApp.id}

+ {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null}
Actions @@ -387,10 +401,6 @@ const Apps = (props) => { })}
: null} - - {activateButton} - {editButton} - {deleteButton}
: null @@ -490,7 +500,7 @@ const Apps = (props) => { setLoadAppsModalOpen(true) }} > - Download from URL + Download more apps
Date: Sun, 24 May 2020 09:02:43 +0200 Subject: [PATCH 09/13] Added switch for searching backend --- frontend/src/Apps.js | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index 8fdde37b..69c5fd02 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -12,6 +12,8 @@ import TextField from '@material-ui/core/TextField'; import FormControl from '@material-ui/core/FormControl'; import MenuItem from '@material-ui/core/MenuItem'; import Tooltip from '@material-ui/core/Tooltip'; +import FormControlLabel from '@material-ui/core/FormControlLabel'; +import Switch from '@material-ui/core/Switch'; import Input from '@material-ui/core/Input'; import YAML from 'yaml' import {Link} from 'react-router-dom'; @@ -43,6 +45,7 @@ const Apps = (props) => { const [isLoading, setIsLoading] = React.useState(false) const [appSearchLoading, setAppSearchLoading] = React.useState(false) const [selectedAction, setSelectedAction] = React.useState({}) + const [searchBackend, setSearchBackend] = React.useState(false) const [openApi, setOpenApi] = React.useState("") const [openApiData, setOpenApiData] = React.useState("") @@ -454,7 +457,8 @@ const Apps = (props) => { const searchfield = event.target.value.toLowerCase() const newapps = apps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) - if (newapps.length === 0 && !appSearchLoading) { + if ((newapps.length === 0 || searchBackend) && !appSearchLoading) { + setAppSearchLoading(true) runAppSearch(searchfield) } else { @@ -477,19 +481,11 @@ const Apps = (props) => {

Available integrations

{isLoading ? : null} - {/* - - */} + Search OpenAPI
+ control={ {setSearchBackend(!searchBackend)}} />} + /> : - + + + +
) @@ -4389,21 +4406,118 @@ const AngularWorkflow = (props) => {
- const ExecutionModal = () => { - return ( - setExecutionModalOpen(false)}> -
+ const executionPaperStyle = { + minWidth: "95%", + maxWidth: "95%", + marginTop: "5px", + color: "white", + marginBottom: 10, + padding: 5, + backgroundColor: surfaceColor, + cursor: "pointer", + display: "flex", + minHeight: 40, + maxHeight: 40, + } + + const executionModal = + setExecutionModalOpen(false)} PaperProps={{style: {minWidth: 375, maxWidth: 375, backgroundColor: "#1F2023", color: "white", fontSize: 18}}}> + {executionModalView === 0 ? +
+ +

+ + All Executions +

+ + + + {workflowExecutions.length > 0 ? +
+ {workflowExecutions.map(data => { + const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" ? "red" : "orange" + const timeElapsed = data.completed_at-data.started_at + const resultsLength = data.results !== undefined && data.results !== null ? data.results.length : 0 + + const timestamp = new Date(data.started_at*1000).toISOString().split('.')[0].split("T").join(" ") + + return ( + {}} onMouseOut={() => {}} onClick={() => { + setExecutionModalView(1) + setExecutionData(data) + }}> +
+
+
+ {timestamp} +
+ +
+ {resultsLength}/{data.workflow.actions.length} +
+ +
+ + + + + ) + return + })} +
+ : +
+ There are no executions yet +
+ } +
+ : +
+ +

{setExecutionModalView(0)}}> + + Other Executions +

+ +

Executing Workflow

- -

Execution Argument:

{executionText} + {executionData.execution_argument !== undefined && executionData.execution_argument.length > 0 ? +
+

Execution Argument:

{executionText} +
+ : null } {executionData.status !== undefined && executionData.status.length > 0 ?
-

Status:

{executionData.status} + Status: {executionData.status}
: null } -

Results:

- {executionData.results === undefined || executionData.results === null || executionData.results.length === 0 ? + {executionData.started_at !== undefined ? +
+ Started: {new Date(executionData.started_at*1000).toISOString()} +
+ : null + } + {executionData.execution_id !== undefined && executionData.execution_id.length > 0 ? +
+ ID: {executionData.execution_id} +
+ : null + } + +
+ Actions +
+ {executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" ? : null} +
+
+ {executionData.results === undefined || executionData.results === null || executionData.results.length === 0 && executionData.status === "EXECUTING" ? : executionData.results.map(data => { @@ -4417,12 +4531,19 @@ const AngularWorkflow = (props) => { jsonvalid = false } + const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) + const imgsize = 50 + const actionimg = curapp === null ? + null : + {data.action.app_name} + return ( -
-
- Name: {data.action.label}
- Status: {data.status}
- Result:  +
+
+ {actionimg} + {data.action.label} +
+
Status {data.status}
{jsonvalid ? { name={"Results for "+data.action.label} /> : - data.result +
+ Result  + {data.result} +
}
) }) }
+ } - ) - } const newView = isLoggedIn ?
@@ -4459,12 +4582,12 @@ const AngularWorkflow = (props) => { }} />
- + e + {executionModal} - {debugView}
:
@@ -4533,7 +4656,7 @@ const AngularWorkflow = (props) => { /> - -
- e {executionModal} diff --git a/frontend/src/App.js b/frontend/src/App.js index 8dc79c44..ac162e47 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -37,11 +37,13 @@ import { createMuiTheme } from '@material-ui/core/styles'; import AlertTemplate from "react-alert-template-basic"; import { positions, Provider } from "react-alert"; -// Testing - localhost -//const globalUrl = "http://192.168.3.6:5001" - // Production - backend proxy forwarding in nginx -const globalUrl = window.location.origin +var globalUrl = window.location.origin +if (window.location.protocol == "http:" && window.location.port === "3000") { + globalUrl = "http://192.168.3.6:5001" +} +console.log(window.location) +console.log(globalUrl) const surfaceColor = "#27292D" const inputColor = "#383B40"