diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index a2276c31..5e4f54c9 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1239,11 +1239,14 @@ class AppBase: print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value]) try: basejson = json.loads(basejson[value]) + print("BASEJSON: %s" % basejson) except json.decoder.JSONDecodeError as e: print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) return basejson[value], False else: basejson = basejson[value] + + print("Parsed BASEJSON: %s" % basejson) outercnt += 1 @@ -1363,6 +1366,7 @@ class AppBase: data, is_loop = recurse_json(basejson, parsersplit[1:]) parseditem = data + print("DATA: %s" % data) if is_loop: print("DATA IS A LOOP - SHOULD WRAP") if parsersplit[-1] == "#": @@ -1415,6 +1419,8 @@ class AppBase: except json.decoder.JSONDecodeError as e: parameter["value"] = parameter["value"].replace(to_be_replaced, value) + print("VALUE: %s" % value) + if parameter["variant"] == "WORKFLOW_VARIABLE": print("Handling workflow variable") @@ -2161,6 +2167,7 @@ class AppBase: print("[INFO] Running normal execution\n") #newres = await func(**params) + print("PARAMS: %s" % params) while True: try: newres = await func(**params) @@ -2181,7 +2188,7 @@ class AppBase: raise e #break - print("\n[INFO] Returned from execution!")#, newres) + print("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres) if isinstance(newres, tuple): print("[INFO] Handling return as tuple") # Handles files. diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 50a73cf7..fdf53a21 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.81 +VERSION=0.8.82 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 7cd34b7d..848fb0ce 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -260,7 +260,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin buildOptions, ) - log.Printf("RESPONSE: %#v", imageBuildResponse) + //log.Printf("RESPONSE: %#v", imageBuildResponse) //log.Printf("Response: %#v", imageBuildResponse.Body) //log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body) @@ -525,7 +525,7 @@ func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) { return } - err := DeleteKey(ctx, "hooks", fileId) + err := shuffle.DeleteKey(ctx, "hooks", fileId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 20ec1d4d..c961e799 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -106,19 +106,6 @@ type ExecutionInfo struct { DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` } -type StatisticsData struct { - Timestamp int64 `json:"timestamp" datastore:"timestamp"` - Id string `json:"id" datastore:"id"` - Amount int64 `json:"amount" datastore:"amount"` -} - -type StatisticsItem struct { - Total int64 `json:"total" datastore:"total"` - Fieldname string `json:"field_name" datastore:"field_name"` - Data []StatisticsData `json:"data" datastore:"data"` - OrgId string `json:"org_id" datastore:"org_id"` -} - // "Execution by status" // Execution history //type GlobalStatistics struct { @@ -593,53 +580,6 @@ func checkFileExistsLocal(basepath string, filepath string) bool { return true } -func handleGetallSchedules(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var err error - var limit = 50 - - // FIXME - add org search and public / private - key, ok := request.URL.Query()["limit"] - if ok { - limit, err = strconv.Atoi(key[0]) - if err != nil { - limit = 50 - } - } - - // Max datastore limit - if limit > 1000 { - limit = 1000 - } - - // Get URLs from a database index (mapped by orborus) - ctx := context.Background() - q := datastore.NewQuery("schedules").Limit(limit) - var allschedules Schedules - - _, err = dbclient.GetAll(ctx, q, &allschedules.Schedules) - if err != nil { - log.Println(err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting schedules"}`))) - return - } - - newjson, err := json.Marshal(allschedules) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) - return - } - - resp.WriteHeader(200) - resp.Write(newjson) -} - func redirect(w http.ResponseWriter, req *http.Request) { // remove/add not default ports from req.Host target := "https://" + req.Host + req.URL.Path @@ -683,166 +623,6 @@ func checkUsername(Username string) error { return nil } -func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - defaultMessage := "Successfully registered" - - var reference string - location := strings.Split(request.URL.String(), "/") - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - reference = location[4] - - if len(reference) != 36 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Id when registering verification is not valid"}`)) - return - } - - ctx := context.Background() - // With user, do a search for workflows with user or user's org attached - // Only giving 200 to not give any suspicion whether they're onto an actual user or not - q := datastore.NewQuery("Users").Filter("verification_token =", reference) - var users []shuffle.User - _, err := dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("Failed getting users for verification token: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) - return - } - - // FIXME - check reset_timeout - if len(users) != 1 { - log.Printf("Error - no user with verification id %s", reference) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - return - } - - Userdata := users[0] - - Userdata.Verified = true - err = shuffle.SetUser(ctx, &Userdata, true) - if err != nil { - log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - log.Printf("[INFO] %s SUCCESSFULLY FINISHED REGISTRATION", Userdata.Username) -} - -func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // FIXME: Overhaul the top part. - // Only admin can change environments, but if there are no users, anyone can make (first) - user, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`)) - return - } - - if user.Role != "admin" { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't set environment without being admin"}`)) - return - } - - ctx := context.Background() - var environments []shuffle.Environment - q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) - _, err = dbclient.GetAll(ctx, q, &environments) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't get environments when setting"}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) - return - } - - var newEnvironments []shuffle.Environment - err = json.Unmarshal(body, &newEnvironments) - if err != nil { - log.Printf("Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) - return - } - - if len(newEnvironments) < 1 { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "One environment is required"}`))) - return - } - - // Clear old data? Removed for archiving purpose. No straight deletion - //for _, item := range environments { - // err = DeleteKey(ctx, "Environments", item.Name) - // if err != nil { - // resp.WriteHeader(401) - // resp.Write([]byte(`{"success": false, "reason": "Error cleaning up environment"}`)) - // return - // } - //} - - openEnvironments := 0 - for _, item := range newEnvironments { - if !item.Archived { - openEnvironments += 1 - } - } - - if openEnvironments < 1 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Can't archived all environments"}`)) - return - } - - for _, item := range newEnvironments { - if item.OrgId != user.ActiveOrg.Id { - item.OrgId = user.ActiveOrg.Id - } - - err = setEnvironment(ctx, &item) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed setting environment variable"}`)) - return - } - } - - //DeleteKey(ctx, entity string, value string) error { - // FIXME - check which are in use - //log.Printf("FIXME: Set new environments: %#v", newEnvironments) - //log.Printf("DONT DELETE ONES THAT ARE IN USE") - - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error { // Returns false if there is an issue // Use this for register @@ -962,7 +742,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can CREATE users, but if there are no users, anyone can make (first) - count, countErr := getUserCount() + count, countErr := shuffle.GetUserCount() user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { if (countErr == nil && count > 0) || countErr != nil { @@ -1038,133 +818,6 @@ func handleCookie(request *http.Request) bool { return true } -func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - userInfo, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in apigen: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field: user_id"}`))) - return - } - - type newUserStruct struct { - Role string `json:"role"` - Username string `json:"username"` - UserId string `json:"user_id"` - } - - ctx := context.Background() - var t newUserStruct - err = json.Unmarshal(body, &t) - if err != nil { - log.Printf("Failed unmarshaling userId: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unmarshaling. Missing field: user_id"}`))) - return - } - - // Should this role reflect the users' org access? - // When you change org -> change user role - if userInfo.Role != "admin" { - log.Printf("%s tried to update user %s", userInfo.Username, t.UserId) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to be admin to change other users"}`))) - return - } - - foundUser, err := shuffle.GetUser(ctx, t.UserId) - if err != nil { - log.Printf("Can't find user %s (update user): %s", t.UserId, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - orgFound := false - for _, item := range foundUser.Orgs { - if item == userInfo.ActiveOrg.Id { - orgFound = true - break - } - } - - if !orgFound { - log.Printf("User %s is admin, but can't edit users outside their own org.", userInfo.Id) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org."}`))) - return - } - - if t.Role != "admin" && t.Role != "user" { - log.Printf("%s tried and failed to update user %s", userInfo.Username, t.UserId) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can only change to role user and admin"}`))) - return - } else { - // Same user - can't edit yourself - if userInfo.Id == t.UserId { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update the role of your own user"}`))) - return - } - - log.Printf("Updated user %s from %s to %s", foundUser.Username, foundUser.Role, t.Role) - foundUser.Role = t.Role - foundUser.Roles = []string{t.Role} - } - - if len(t.Username) > 0 { - q := datastore.NewQuery("Users").Filter("username =", t.Username) - var users []shuffle.User - _, err = dbclient.GetAll(ctx, q, &users) - if err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed getting users when updating user"}`)) - return - } - - found := false - for _, item := range users { - if item.Username == t.Username { - found = true - break - } - } - - if found { - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "User with username %s already exists"}`, t.Username))) - return - } - - foundUser.Username = t.Username - } - - err = shuffle.SetUser(ctx, foundUser, true) - if err != nil { - log.Printf("Error patching user %s: %s", foundUser.Username, err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - func handleInfo(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -1350,94 +1003,6 @@ type passwordReset struct { Reference string `json:"reference"` } -func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - log.Println("Handling password reset") - defaultMessage := "Successfully handled password reset" - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - var t passwordReset - err = json.Unmarshal(body, &t) - if err != nil { - log.Println("Failed unmarshaling") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - return - } - - if t.Password1 != t.Password2 { - resp.WriteHeader(401) - err := "Passwords don't match" - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - if len(t.Password1) < 10 || len(t.Password2) < 10 { - resp.WriteHeader(401) - err := "Passwords don't match - 2" - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) - return - } - - ctx := context.Background() - - // With user, do a search for workflows with user or user's org attached - // Only giving 200 to not give any suspicion whether they're onto an actual user or not - q := datastore.NewQuery("Users").Filter("reset_reference =", t.Reference) - var users []shuffle.User - _, err = dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("Failed getting users: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) - return - } - - // FIXME - check reset_timeout - if len(users) != 1 { - log.Printf("Error - no user with id %s", t.Reference) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - return - } - - Userdata := users[0] - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Password1), 8) - if err != nil { - log.Printf("Wrong password for %s: %s", Userdata.Username, err) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - return - } - - Userdata.Password = string(hashedPassword) - Userdata.ResetTimeout = 0 - Userdata.ResetReference = "" - err = shuffle.SetUser(ctx, &Userdata, true) - if err != nil { - log.Printf("Error adding User %s: %s", Userdata.Username, err) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) - return - } - - // FIXME - maybe send a mail here to say that the password was changed - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) -} - // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -1485,35 +1050,13 @@ func handleContact(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`))) } -func getEnvironmentCount() (int, error) { - ctx := context.Background() - q := datastore.NewQuery("Environments").Limit(1) - count, err := dbclient.Count(ctx, q) - if err != nil { - return 0, err - } - - return count, nil -} - -func getUserCount() (int, error) { - ctx := context.Background() - q := datastore.NewQuery("Users").Limit(1) - count, err := dbclient.Count(ctx, q) - if err != nil { - return 0, err - } - - return count, nil -} - func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - count, err := getUserCount() + count, err := shuffle.GetUserCount() if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) @@ -1648,37 +1191,6 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(loginData)) } -// Index = Username -func DeleteKeys(ctx context.Context, entity string, value []string) error { - // Non indexed User data - keys := []*datastore.Key{} - for _, item := range value { - keys = append(keys, datastore.NameKey(entity, item, nil)) - } - - err := dbclient.DeleteMulti(ctx, keys) - if err != nil { - log.Printf("Error deleting %s from %s: %s", value, entity, err) - return err - } - - return nil -} - -// Index = Username -func DeleteKey(ctx context.Context, entity string, value string) error { - // Non indexed User data - key1 := datastore.NameKey(entity, value, nil) - - err := dbclient.Delete(ctx, key1) - if err != nil { - log.Printf("Error deleting %s from %s: %s", value, entity, err) - return err - } - - return nil -} - func setOpenApiDatastore(ctx context.Context, id string, data ParsedOpenApi) error { k := datastore.NameKey("openapi3", id, nil) if _, err := dbclient.Put(ctx, k, &data); err != nil { @@ -2335,7 +1847,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - err = DeleteKey(ctx, "schedules", workflowId) + err = shuffle.DeleteKey(ctx, "schedules", workflowId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) @@ -2484,7 +1996,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { if err == nil { for _, branch := range workflow.Branches { if branch.SourceID == hook.Id { - log.Printf("Found ID %s for hook", hook.Id) + log.Printf("[INFO] Found ID %s for hook", hook.Id) if branch.DestinationID != hook.Start { newBody.Start = branch.DestinationID break @@ -3636,55 +3148,6 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { resp.Write(b) } -func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - _, err := shuffle.HandleApiAuthentication(resp, request) - if err != nil { - log.Printf("Api authentication failed in getting specific workflow: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - location := strings.Split(request.URL.String(), "/") - - var statsId string - if location[1] == "api" { - if len(location) <= 4 { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - statsId = location[4] - } - - ctx := context.Background() - statisticsId := "global_statistics" - nameKey := statsId - key := datastore.NameKey(statisticsId, nameKey, nil) - statisticsItem := StatisticsItem{} - if err := dbclient.Get(ctx, key, &statisticsItem); err != nil { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - b, err := json.Marshal(statisticsItem) - if err != nil { - log.Printf("Failed to marshal data: %s", err) - resp.WriteHeader(401) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(b)) -} - func getOpenapi(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -4893,7 +4356,7 @@ func runInit(ctx context.Context) { for _, user := range users { user.Active = true if len(user.Username) == 0 { - DeleteKey(ctx, "Users", strings.ToLower(user.Username)) + shuffle.DeleteKey(ctx, "Users", strings.ToLower(user.Username)) continue } @@ -4915,7 +4378,7 @@ func runInit(ctx context.Context) { log.Printf("Failed to reset user") } else { log.Printf("Remade user %s with ID", user.Id) - err = DeleteKey(ctx, "Users", strings.ToLower(user.Username)) + err = shuffle.DeleteKey(ctx, "Users", strings.ToLower(user.Username)) if err != nil { log.Printf("Failed to delete old user by username") } @@ -4973,7 +4436,7 @@ func runInit(ctx context.Context) { } // Gets environments and inits if it doesn't exist - count, err := getEnvironmentCount() + count, err := shuffle.GetEnvironmentCount() if count == 0 && err == nil && len(activeOrgs) == 1 { log.Printf("[INFO] Setting up environment with org %s", activeOrgs[0].Id) item := shuffle.Environment{ @@ -5957,7 +5420,7 @@ func initHandlers() { r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/users/updateuser", handleUpdateUser).Methods("PUT", "OPTIONS") + r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/users", shuffle.HandleGetUsers).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index ac9e766a..64cec153 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -487,8 +487,8 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i nameKey := fieldname key := datastore.NameKey(statisticsId, nameKey, nil) - statisticsItem := StatisticsItem{} - newData := StatisticsData{ + statisticsItem := shuffle.StatisticsItem{} + newData := shuffle.StatisticsData{ Timestamp: int64(time.Now().Unix()), Amount: amount, Id: id, @@ -497,11 +497,11 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i if err := dbclient.Get(ctx, key, &statisticsItem); err != nil { // Should init if strings.Contains(fmt.Sprintf("%s", err), "entity") { - statisticsItem = StatisticsItem{ + statisticsItem = shuffle.StatisticsItem{ Total: amount, OrgId: orgId, Fieldname: fieldname, - Data: []StatisticsData{ + Data: []shuffle.StatisticsData{ newData, }, } @@ -752,7 +752,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque ids = append(ids, execution.ExecutionId) } - err = DeleteKeys(ctx, parsedId, ids) + err = shuffle.DeleteKeys(ctx, parsedId, ids) if err != nil { log.Printf("[ERROR] Failed deleting %d execution keys for org %s", len(ids), id) } else { @@ -796,7 +796,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { id := request.Header.Get("Org-Id") if len(id) == 0 { - log.Printf("No org-id header set") + log.Printf("[INFO] No org-id header set") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) return @@ -843,10 +843,11 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } + //log.Printf("Data: %s", string(body)) var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { - log.Printf("Failed ActionResult unmarshaling: %s", err) + log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return @@ -863,7 +864,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // Authorization is done here if workflowExecution.Authorization != actionResult.Authorization { - log.Printf("Bad authorization key when getting stream results %s.", actionResult.ExecutionId) + log.Printf("[WARNING] Bad authorization key when getting stream results %s.", actionResult.ExecutionId) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return @@ -941,7 +942,7 @@ func validateNewWorkerExecution(body []byte) error { //log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n") err = shuffle.SetWorkflowExecution(ctx, execution, true) if err == nil { - log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results)) + log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d, Status: %s, Result: %s", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status, execution.Result) //log.Printf("[INFO] Successfully set the execution to wait.") } else { log.Printf("[WARNING] Failed to set the execution to wait.") @@ -977,7 +978,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { - log.Printf("Failed ActionResult unmarshaling: %s", err) + log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return @@ -1007,15 +1008,13 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { return } - if workflowExecution.Status == "FINISHED" { - log.Printf("[INFO] Workflowexecution is already FINISHED. No further action can be taken.") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) - return - } + //if workflowExecution.Status == "FINISHED" { + // log.Printf("[INFO] Workflowexecution is already FINISHED. No further action can be taken.") + // resp.WriteHeader(401) + // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) + // return + //} - // Not sure what's up here - // FIXME - remove comment if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { if workflowExecution.Workflow.Configuration.ExitOnError { @@ -1283,7 +1282,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { } } else { log.Printf("Failed getting workflows for user %s: %s (1)", user.Username, err) - //DeleteKey(ctx, "workflow", "5694357e-8063-4580-8529-301cc72df951") + //shuffle.DeleteKey(ctx, "workflow", "5694357e-8063-4580-8529-301cc72df951") //log.Printf("Workflows: %#v", workflows) resp.WriteHeader(401) @@ -1404,7 +1403,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - maybe delete workflow executions log.Printf("[INFO] Should have deleted workflow %s", fileId) - err = DeleteKey(ctx, "workflow", fileId) + err = shuffle.DeleteKey(ctx, "workflow", fileId) if err != nil { log.Printf("Failed deleting key %s", fileId) resp.WriteHeader(401) @@ -2511,7 +2510,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } else { log.Printf("[INFO] Successfully ran cloud action STOP schedule") - err = DeleteKey(ctx, "schedules", scheduleId) + err = shuffle.DeleteKey(ctx, "schedules", scheduleId) if err != nil { log.Printf("[WARNING] Failed deleting cloud schedule onprem..") resp.WriteHeader(401) @@ -2634,7 +2633,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { func deleteSchedule(ctx context.Context, id string) error { log.Printf("Should stop schedule %s!", id) - err := DeleteKey(ctx, "schedules", id) + err := shuffle.DeleteKey(ctx, "schedules", id) if err != nil { log.Printf("Failed to delete schedule: %s", err) return err @@ -3054,7 +3053,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] Deleting public app") - err = DeleteKey(ctx, "workflowapp", fileId) + err = shuffle.DeleteKey(ctx, "workflowapp", fileId) if err != nil { log.Printf("Failed deleting workflowapp") resp.WriteHeader(401) @@ -4247,7 +4246,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(removeApps) > 0 { for _, item := range removeApps { log.Printf("[WARNING] Removing duplicate app: %s", item) - err = DeleteKey(ctx, "workflowapp", item) + err = shuffle.DeleteKey(ctx, "workflowapp", item) if err != nil { log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err) } @@ -4950,3 +4949,20 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId return nil } + +// Index = Username +func DeleteKeys(ctx context.Context, entity string, value []string) error { + // Non indexed User data + keys := []*datastore.Key{} + for _, item := range value { + keys = append(keys, datastore.NameKey(entity, item, nil)) + } + + err := dbclient.DeleteMulti(ctx, keys) + if err != nil { + log.Printf("Error deleting %s from %s: %s", value, entity, err) + return err + } + + return nil +} diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 7f11f8d4..82998cf8 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.80 +VERSION=0.8.81 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 28f35a5a..efef4ddb 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -244,11 +244,11 @@ func initializeImages() { ctx := context.Background() if appSdkVersion == "" { - appSdkVersion = "0.8.80" + appSdkVersion = "0.8.82" log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.80" + workerVersion = "0.8.82" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 6ef340fa..3b8218a1 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,20 +1,25 @@ FROM golang:1.16.0-buster as builder WORKDIR /app -RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client +#RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client #RUN go env -w GO111MODULE=auto COPY worker.go /app/worker.go -RUN go mod init worker -RUN go get github.com/docker/docker/api/types && \ - go get github.com/docker/docker/api/types/container && \ - go get github.com/docker/docker/client && \ - go get github.com/gorilla/mux && \ - go get github.com/patrickmn/go-cache && \ - go get github.com/frikky/shuffle-shared && \ - go get github.com/satori/go.uuid && \ - go get github.com/fsouza/go-dockerclient && \ - go get google.golang.org/grpc/balancer/grpclb@v1.37.1 +COPY go.mod /app/go.mod +COPY go.sum /app/go.sum +RUN go get + +#RUN go mod init worker +#RUN go get +#RUN go get github.com/docker/docker/api/types && \ +# go get github.com/docker/docker/api/types/container && \ +# go get github.com/docker/docker/client && \ +# go get github.com/gorilla/mux && \ +# go get github.com/patrickmn/go-cache && \ +# go get github.com/frikky/shuffle-shared && \ +# go get github.com/satori/go.uuid && \ +# go get github.com/fsouza/go-dockerclient && \ +# go get google.golang.org/grpc/balancer/grpclb@v1.37.1 RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index d38e48c6..bc98208a 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.80 +VERSION=0.8.82 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 651b1514..6d099c67 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -12,7 +12,7 @@ require ( github.com/docker/docker v20.10.5+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/frikky/shuffle-shared v0.0.40 + github.com/frikky/shuffle-shared v0.0.45 github.com/fsouza/go-dockerclient v1.7.2 github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 4b39b0ee..e5b79140 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -133,7 +133,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv) } - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) + abortUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) path := fmt.Sprintf("?reason=%s", url.QueryEscape(reason)) if len(nodeId) > 0 { @@ -144,12 +144,12 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason } //fmt.Println(url.QueryEscape(query)) - fullUrl += path - log.Printf("[INFO] Abort URL: %s", fullUrl) + abortUrl += path + log.Printf("[INFO] Abort URL: %s", abortUrl) req, err := http.NewRequest( "GET", - fullUrl, + abortUrl, nil, ) @@ -426,15 +426,15 @@ func handleSubworkflowExecution(client *http.Client, workflowExecution shuffle.W baseResult = `{"success": false}` } else { log.Printf("Should execute workflow %s with APIKEY %s and data %s", workflowId, apikey, executionArgument) - fullUrl := fmt.Sprintf("%s/api/workflows/%s/execute", baseUrl, workflowId) + executeUrl := fmt.Sprintf("%s/api/workflows/%s/execute", baseUrl, workflowId) req, err := http.NewRequest( "POST", - fullUrl, + executeUrl, bytes.NewBuffer([]byte(executionArgument)), ) if err != nil { - log.Printf("Error building test request: %s", err) + log.Printf("[WARNING] Error building test request: %s", err) return err } @@ -477,10 +477,10 @@ func handleSubworkflowExecution(client *http.Client, workflowExecution shuffle.W return err } - fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", - fullUrl, + streamUrl, bytes.NewBuffer([]byte(resultData)), ) @@ -801,8 +801,10 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { // Execute, as we don't really care if env is not set? IDK if action.Environment != environment { //&& action.Environment != "" { //log.Printf("Action: %#v", action) - log.Printf("Bad environment for node: %s. Want %s", action.Environment, environment) - continue + log.Printf("[WARNING] Bad environment for node: %#v. Want %s. Skipping if NOT empty env.", action.Environment, environment) + if len(action.Environment) > 0 { + continue + } } // check whether the parent is finished executing @@ -975,41 +977,47 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } - log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") image = images[2] - reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) - if err != nil { - log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) - } - - buildBuf := new(strings.Builder) - _, err = io.Copy(buildBuf, reader) - if err != nil && !strings.Contains(fmt.Sprintf("Docker error: %s", err.Error()), "Conflict. The container name") { - log.Printf("[ERROR] Error in IO copy: %s", err) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) - } else { - if strings.Contains(buildBuf.String(), "errorDetail") { - log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) - shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) - } - - log.Printf("[INFO] Successfully downloaded %s", image) - } - err = deployApp(dockercli, image, identifier, env, workflowExecution) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { - - log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") if strings.Contains(err.Error(), "exited prematurely") { shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } - - if strings.Contains(err.Error(), "No such image") { - //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) - log.Printf("[ERROR] Image doesn't exist. Shutting down") + log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.") + reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image) shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) } + + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, reader) + if err != nil && !strings.Contains(fmt.Sprintf("Docker error: %s", err.Error()), "Conflict. The container name") { + log.Printf("[ERROR] Error in IO copy: %s", err) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + } + + log.Printf("[INFO] Successfully downloaded %s", image) + } + + err = deployApp(dockercli, image, identifier, env, workflowExecution) + if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { + + log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "exited prematurely") { + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + } + + if strings.Contains(err.Error(), "No such image") { + //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true) + } + } } } } else { @@ -1253,13 +1261,13 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec ctx := context.Background() setWorkflowExecution(ctx, workflowExecution, false) + streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) for { //fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId) - fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) //log.Printf("[INFO] URL: %s", fullUrl) req, err := http.NewRequest( "POST", - fullUrl, + streamResultUrl, bytes.NewBuffer([]byte(data)), ) @@ -1375,10 +1383,10 @@ func runUserInput(client *http.Client, action shuffle.Action, workflowId, workfl return err } - fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", - fullUrl, + streamUrl, bytes.NewBuffer([]byte(resultData)), ) @@ -1404,10 +1412,10 @@ func runUserInput(client *http.Client, action shuffle.Action, workflowId, workfl } func runTestExecution(client *http.Client, workflowId, apikey string) (string, string) { - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", baseUrl, workflowId) + executeUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", baseUrl, workflowId) req, err := http.NewRequest( "GET", - fullUrl, + executeUrl, nil, ) @@ -1484,12 +1492,10 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if workflowExecution.Status == "FINISHED" { log.Printf("Workflowexecution is already FINISHED. No further action can be taken") resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because it has status %s"}`, workflowExecution.LastNode, workflowExecution.Status))) return } - // Not sure what's up here - // FIXME - remove comment if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" { if workflowExecution.Workflow.Configuration.ExitOnError { @@ -1602,10 +1608,10 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) { return } - fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) + streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl) req, err := http.NewRequest( "POST", - fullUrl, + streamUrl, bytes.NewBuffer([]byte(data)), ) @@ -1777,11 +1783,11 @@ func runWebserver(listener net.Listener) { func downloadDockerImage(client *http.Client, imageName string) { data := fmt.Sprintf(`{"name": "%s"}`, imageName) - fullUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl) + dockerImgUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl) req, err := http.NewRequest( "POST", - fullUrl, + dockerImgUrl, bytes.NewBuffer([]byte(data)), ) @@ -1862,7 +1868,6 @@ func downloadDockerImage(client *http.Client, imageName string) { // Initial loop etc func main() { - log.Printf("[INFO] Setting up worker environment") sleepTime := 5 @@ -1920,10 +1925,10 @@ func main() { } data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization) - fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) + streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl) req, err := http.NewRequest( "POST", - fullUrl, + streamResultUrl, bytes.NewBuffer([]byte(data)), )