marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Started Opensearch migration

Browse Source
This commit is contained in:
frikky
2021-05-20 16:23:15 +02:00
parent 892fc768eb
commit 79b2d83dd4
11 changed files with 139 additions and 643 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+8 -1
View File
@@ -1239,11 +1239,14 @@ class AppBase:
print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value])
try:
basejson = json.loads(basejson[value])
print("BASEJSON: %s" % basejson)
except json.decoder.JSONDecodeError as e:
print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value])
return basejson[value], False
else:
basejson = basejson[value]
print("Parsed BASEJSON: %s" % basejson)
outercnt += 1
@@ -1363,6 +1366,7 @@ class AppBase:
data, is_loop = recurse_json(basejson, parsersplit[1:])
parseditem = data
print("DATA: %s" % data)
if is_loop:
print("DATA IS A LOOP - SHOULD WRAP")
if parsersplit[-1] == "#":
@@ -1415,6 +1419,8 @@ class AppBase:
except json.decoder.JSONDecodeError as e:
parameter["value"] = parameter["value"].replace(to_be_replaced, value)
print("VALUE: %s" % value)
if parameter["variant"] == "WORKFLOW_VARIABLE":
print("Handling workflow variable")
@@ -2161,6 +2167,7 @@ class AppBase:
print("[INFO] Running normal execution\n")
#newres = await func(**params)
print("PARAMS: %s" % params)
while True:
try:
newres = await func(**params)
@@ -2181,7 +2188,7 @@ class AppBase:
raise e
#break
print("\n[INFO] Returned from execution!")#, newres)
print("\n[INFO] Returned from execution with %s of types %s" % (newres, type(newres)))#, newres)
if isinstance(newres, tuple):
print("[INFO] Handling return as tuple")
# Handles files.
backend/app_sdk/build.sh
+1 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.81
VERSION=0.8.82
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
backend/go-app/docker.go
+2 -2
View File
@@ -260,7 +260,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
buildOptions,
)
log.Printf("RESPONSE: %#v", imageBuildResponse)
//log.Printf("RESPONSE: %#v", imageBuildResponse)
//log.Printf("Response: %#v", imageBuildResponse.Body)
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
@@ -525,7 +525,7 @@ func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) {
return
}
err := DeleteKey(ctx, "hooks", fileId)
err := shuffle.DeleteKey(ctx, "hooks", fileId)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "Can't delete"}`))
backend/go-app/main.go
+8 -545
View File
@@ -106,19 +106,6 @@ type ExecutionInfo struct {
DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"`
}
type StatisticsData struct {
Timestamp int64 `json:"timestamp" datastore:"timestamp"`
Id string `json:"id" datastore:"id"`
Amount int64 `json:"amount" datastore:"amount"`
}
type StatisticsItem struct {
Total int64 `json:"total" datastore:"total"`
Fieldname string `json:"field_name" datastore:"field_name"`
Data []StatisticsData `json:"data" datastore:"data"`
OrgId string `json:"org_id" datastore:"org_id"`
}
// "Execution by status"
// Execution history
//type GlobalStatistics struct {
@@ -593,53 +580,6 @@ func checkFileExistsLocal(basepath string, filepath string) bool {
return true
}
func handleGetallSchedules(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
var err error
var limit = 50
// FIXME - add org search and public / private
key, ok := request.URL.Query()["limit"]
if ok {
limit, err = strconv.Atoi(key[0])
if err != nil {
limit = 50
}
}
// Max datastore limit
if limit > 1000 {
limit = 1000
}
// Get URLs from a database index (mapped by orborus)
ctx := context.Background()
q := datastore.NewQuery("schedules").Limit(limit)
var allschedules Schedules
_, err = dbclient.GetAll(ctx, q, &allschedules.Schedules)
if err != nil {
log.Println(err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting schedules"}`)))
return
}
newjson, err := json.Marshal(allschedules)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func redirect(w http.ResponseWriter, req *http.Request) {
// remove/add not default ports from req.Host
target := "https://" + req.Host + req.URL.Path
@@ -683,166 +623,6 @@ func checkUsername(Username string) error {
return nil
}
func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
defaultMessage := "Successfully registered"
var reference string
location := strings.Split(request.URL.String(), "/")
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
reference = location[4]
if len(reference) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Id when registering verification is not valid"}`))
return
}
ctx := context.Background()
// With user, do a search for workflows with user or user's org attached
// Only giving 200 to not give any suspicion whether they're onto an actual user or not
q := datastore.NewQuery("Users").Filter("verification_token =", reference)
var users []shuffle.User
_, err := dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Failed getting users for verification token: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage)))
return
}
// FIXME - check reset_timeout
if len(users) != 1 {
log.Printf("Error - no user with verification id %s", reference)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
return
}
Userdata := users[0]
Userdata.Verified = true
err = shuffle.SetUser(ctx, &Userdata, true)
if err != nil {
log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
log.Printf("[INFO] %s SUCCESSFULLY FINISHED REGISTRATION", Userdata.Username)
}
func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// FIXME: Overhaul the top part.
// Only admin can change environments, but if there are no users, anyone can make (first)
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't set environment without being admin"}`))
return
}
ctx := context.Background()
var environments []shuffle.Environment
q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id)
_, err = dbclient.GetAll(ctx, q, &environments)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't get environments when setting"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Println("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`)))
return
}
var newEnvironments []shuffle.Environment
err = json.Unmarshal(body, &newEnvironments)
if err != nil {
log.Printf("Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`)))
return
}
if len(newEnvironments) < 1 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "One environment is required"}`)))
return
}
// Clear old data? Removed for archiving purpose. No straight deletion
//for _, item := range environments {
// err = DeleteKey(ctx, "Environments", item.Name)
// if err != nil {
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false, "reason": "Error cleaning up environment"}`))
// return
// }
//}
openEnvironments := 0
for _, item := range newEnvironments {
if !item.Archived {
openEnvironments += 1
}
}
if openEnvironments < 1 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't archived all environments"}`))
return
}
for _, item := range newEnvironments {
if item.OrgId != user.ActiveOrg.Id {
item.OrgId = user.ActiveOrg.Id
}
err = setEnvironment(ctx, &item)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed setting environment variable"}`))
return
}
}
//DeleteKey(ctx, entity string, value string) error {
// FIXME - check which are in use
//log.Printf("FIXME: Set new environments: %#v", newEnvironments)
//log.Printf("DONT DELETE ONES THAT ARE IN USE")
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error {
// Returns false if there is an issue
// Use this for register
@@ -962,7 +742,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) {
// FIXME: Overhaul the top part.
// Only admin can CREATE users, but if there are no users, anyone can make (first)
count, countErr := getUserCount()
count, countErr := shuffle.GetUserCount()
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
if (countErr == nil && count > 0) || countErr != nil {
@@ -1038,133 +818,6 @@ func handleCookie(request *http.Request) bool {
return true
}
func handleUpdateUser(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
userInfo, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in apigen: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Println("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field: user_id"}`)))
return
}
type newUserStruct struct {
Role string `json:"role"`
Username string `json:"username"`
UserId string `json:"user_id"`
}
ctx := context.Background()
var t newUserStruct
err = json.Unmarshal(body, &t)
if err != nil {
log.Printf("Failed unmarshaling userId: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unmarshaling. Missing field: user_id"}`)))
return
}
// Should this role reflect the users' org access?
// When you change org -> change user role
if userInfo.Role != "admin" {
log.Printf("%s tried to update user %s", userInfo.Username, t.UserId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to be admin to change other users"}`)))
return
}
foundUser, err := shuffle.GetUser(ctx, t.UserId)
if err != nil {
log.Printf("Can't find user %s (update user): %s", t.UserId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
orgFound := false
for _, item := range foundUser.Orgs {
if item == userInfo.ActiveOrg.Id {
orgFound = true
break
}
}
if !orgFound {
log.Printf("User %s is admin, but can't edit users outside their own org.", userInfo.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org."}`)))
return
}
if t.Role != "admin" && t.Role != "user" {
log.Printf("%s tried and failed to update user %s", userInfo.Username, t.UserId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can only change to role user and admin"}`)))
return
} else {
// Same user - can't edit yourself
if userInfo.Id == t.UserId {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update the role of your own user"}`)))
return
}
log.Printf("Updated user %s from %s to %s", foundUser.Username, foundUser.Role, t.Role)
foundUser.Role = t.Role
foundUser.Roles = []string{t.Role}
}
if len(t.Username) > 0 {
q := datastore.NewQuery("Users").Filter("username =", t.Username)
var users []shuffle.User
_, err = dbclient.GetAll(ctx, q, &users)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting users when updating user"}`))
return
}
found := false
for _, item := range users {
if item.Username == t.Username {
found = true
break
}
}
if found {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "User with username %s already exists"}`, t.Username)))
return
}
foundUser.Username = t.Username
}
err = shuffle.SetUser(ctx, foundUser, true)
if err != nil {
log.Printf("Error patching user %s: %s", foundUser.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleInfo(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -1350,94 +1003,6 @@ type passwordReset struct {
Reference string `json:"reference"`
}
func handlePasswordReset(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
log.Println("Handling password reset")
defaultMessage := "Successfully handled password reset"
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Println("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
var t passwordReset
err = json.Unmarshal(body, &t)
if err != nil {
log.Println("Failed unmarshaling")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
if t.Password1 != t.Password2 {
resp.WriteHeader(401)
err := "Passwords don't match"
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
if len(t.Password1) < 10 || len(t.Password2) < 10 {
resp.WriteHeader(401)
err := "Passwords don't match - 2"
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
ctx := context.Background()
// With user, do a search for workflows with user or user's org attached
// Only giving 200 to not give any suspicion whether they're onto an actual user or not
q := datastore.NewQuery("Users").Filter("reset_reference =", t.Reference)
var users []shuffle.User
_, err = dbclient.GetAll(ctx, q, &users)
if err != nil {
log.Printf("Failed getting users: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage)))
return
}
// FIXME - check reset_timeout
if len(users) != 1 {
log.Printf("Error - no user with id %s", t.Reference)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
return
}
Userdata := users[0]
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Password1), 8)
if err != nil {
log.Printf("Wrong password for %s: %s", Userdata.Username, err)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
return
}
Userdata.Password = string(hashedPassword)
Userdata.ResetTimeout = 0
Userdata.ResetReference = ""
err = shuffle.SetUser(ctx, &Userdata, true)
if err != nil {
log.Printf("Error adding User %s: %s", Userdata.Username, err)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
return
}
// FIXME - maybe send a mail here to say that the password was changed
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage)))
}
// FIXME - forward this to emails or whatever CRM system in use
func handleContact(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
@@ -1485,35 +1050,13 @@ func handleContact(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`)))
}
func getEnvironmentCount() (int, error) {
ctx := context.Background()
q := datastore.NewQuery("Environments").Limit(1)
count, err := dbclient.Count(ctx, q)
if err != nil {
return 0, err
}
return count, nil
}
func getUserCount() (int, error) {
ctx := context.Background()
q := datastore.NewQuery("Users").Limit(1)
count, err := dbclient.Count(ctx, q)
if err != nil {
return 0, err
}
return count, nil
}
func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
count, err := getUserCount()
count, err := shuffle.GetUserCount()
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
@@ -1648,37 +1191,6 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(loginData))
}
// Index = Username
func DeleteKeys(ctx context.Context, entity string, value []string) error {
// Non indexed User data
keys := []*datastore.Key{}
for _, item := range value {
keys = append(keys, datastore.NameKey(entity, item, nil))
}
err := dbclient.DeleteMulti(ctx, keys)
if err != nil {
log.Printf("Error deleting %s from %s: %s", value, entity, err)
return err
}
return nil
}
// Index = Username
func DeleteKey(ctx context.Context, entity string, value string) error {
// Non indexed User data
key1 := datastore.NameKey(entity, value, nil)
err := dbclient.Delete(ctx, key1)
if err != nil {
log.Printf("Error deleting %s from %s: %s", value, entity, err)
return err
}
return nil
}
func setOpenApiDatastore(ctx context.Context, id string, data ParsedOpenApi) error {
k := datastore.NameKey("openapi3", id, nil)
if _, err := dbclient.Put(ctx, k, &data); err != nil {
@@ -2335,7 +1847,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) {
}
ctx := context.Background()
err = DeleteKey(ctx, "schedules", workflowId)
err = shuffle.DeleteKey(ctx, "schedules", workflowId)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "Can't delete"}`))
@@ -2484,7 +1996,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
if err == nil {
for _, branch := range workflow.Branches {
if branch.SourceID == hook.Id {
log.Printf("Found ID %s for hook", hook.Id)
log.Printf("[INFO] Found ID %s for hook", hook.Id)
if branch.DestinationID != hook.Start {
newBody.Start = branch.DestinationID
break
@@ -3636,55 +3148,6 @@ func getDocs(resp http.ResponseWriter, request *http.Request) {
resp.Write(b)
}
func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
_, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in getting specific workflow: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var statsId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
statsId = location[4]
}
ctx := context.Background()
statisticsId := "global_statistics"
nameKey := statsId
key := datastore.NameKey(statisticsId, nameKey, nil)
statisticsItem := StatisticsItem{}
if err := dbclient.Get(ctx, key, &statisticsItem); err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
b, err := json.Marshal(statisticsItem)
if err != nil {
log.Printf("Failed to marshal data: %s", err)
resp.WriteHeader(401)
return
}
resp.WriteHeader(200)
resp.Write([]byte(b))
}
func getOpenapi(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -4893,7 +4356,7 @@ func runInit(ctx context.Context) {
for _, user := range users {
user.Active = true
if len(user.Username) == 0 {
DeleteKey(ctx, "Users", strings.ToLower(user.Username))
shuffle.DeleteKey(ctx, "Users", strings.ToLower(user.Username))
continue
}
@@ -4915,7 +4378,7 @@ func runInit(ctx context.Context) {
log.Printf("Failed to reset user")
} else {
log.Printf("Remade user %s with ID", user.Id)
err = DeleteKey(ctx, "Users", strings.ToLower(user.Username))
err = shuffle.DeleteKey(ctx, "Users", strings.ToLower(user.Username))
if err != nil {
log.Printf("Failed to delete old user by username")
}
@@ -4973,7 +4436,7 @@ func runInit(ctx context.Context) {
}
// Gets environments and inits if it doesn't exist
count, err := getEnvironmentCount()
count, err := shuffle.GetEnvironmentCount()
if count == 0 && err == nil && len(activeOrgs) == 1 {
log.Printf("[INFO] Setting up environment with org %s", activeOrgs[0].Id)
item := shuffle.Environment{
@@ -5957,7 +5420,7 @@ func initHandlers() {
r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/updateuser", handleUpdateUser).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
backend/go-app/walkoff.go
+40 -24
View File
@@ -487,8 +487,8 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i
nameKey := fieldname
key := datastore.NameKey(statisticsId, nameKey, nil)
statisticsItem := StatisticsItem{}
newData := StatisticsData{
statisticsItem := shuffle.StatisticsItem{}
newData := shuffle.StatisticsData{
Timestamp: int64(time.Now().Unix()),
Amount: amount,
Id: id,
@@ -497,11 +497,11 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i
if err := dbclient.Get(ctx, key, &statisticsItem); err != nil {
// Should init
if strings.Contains(fmt.Sprintf("%s", err), "entity") {
statisticsItem = StatisticsItem{
statisticsItem = shuffle.StatisticsItem{
Total: amount,
OrgId: orgId,
Fieldname: fieldname,
Data: []StatisticsData{
Data: []shuffle.StatisticsData{
newData,
},
}
@@ -752,7 +752,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
ids = append(ids, execution.ExecutionId)
}
err = DeleteKeys(ctx, parsedId, ids)
err = shuffle.DeleteKeys(ctx, parsedId, ids)
if err != nil {
log.Printf("[ERROR] Failed deleting %d execution keys for org %s", len(ids), id)
} else {
@@ -796,7 +796,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
id := request.Header.Get("Org-Id")
if len(id) == 0 {
log.Printf("No org-id header set")
log.Printf("[INFO] No org-id header set")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`)))
return
@@ -843,10 +843,11 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) {
return
}
//log.Printf("Data: %s", string(body))
var actionResult shuffle.ActionResult
err = json.Unmarshal(body, &actionResult)
if err != nil {
log.Printf("Failed ActionResult unmarshaling: %s", err)
log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
@@ -863,7 +864,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) {
// Authorization is done here
if workflowExecution.Authorization != actionResult.Authorization {
log.Printf("Bad authorization key when getting stream results %s.", actionResult.ExecutionId)
log.Printf("[WARNING] Bad authorization key when getting stream results %s.", actionResult.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`)))
return
@@ -941,7 +942,7 @@ func validateNewWorkerExecution(body []byte) error {
//log.Printf("\n\nSHOULD SET BACKEND DATA FOR EXEC \n\n")
err = shuffle.SetWorkflowExecution(ctx, execution, true)
if err == nil {
log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results))
log.Printf("[INFO] Set workflowexecution based on new worker (>0.8.53) for execution %s. Actions: %d, Triggers: %d, Results: %d, Status: %s, Result: %s", execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status, execution.Result)
//log.Printf("[INFO] Successfully set the execution to wait.")
} else {
log.Printf("[WARNING] Failed to set the execution to wait.")
@@ -977,7 +978,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
var actionResult shuffle.ActionResult
err = json.Unmarshal(body, &actionResult)
if err != nil {
log.Printf("Failed ActionResult unmarshaling: %s", err)
log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
@@ -1007,15 +1008,13 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
return
}
if workflowExecution.Status == "FINISHED" {
log.Printf("[INFO] Workflowexecution is already FINISHED. No further action can be taken.")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status)))
return
}
//if workflowExecution.Status == "FINISHED" {
// log.Printf("[INFO] Workflowexecution is already FINISHED. No further action can be taken.")
// resp.WriteHeader(401)
// resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status)))
// return
//}
// Not sure what's up here
// FIXME - remove comment
if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" {
if workflowExecution.Workflow.Configuration.ExitOnError {
@@ -1283,7 +1282,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
}
} else {
log.Printf("Failed getting workflows for user %s: %s (1)", user.Username, err)
//DeleteKey(ctx, "workflow", "5694357e-8063-4580-8529-301cc72df951")
//shuffle.DeleteKey(ctx, "workflow", "5694357e-8063-4580-8529-301cc72df951")
//log.Printf("Workflows: %#v", workflows)
resp.WriteHeader(401)
@@ -1404,7 +1403,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
// FIXME - maybe delete workflow executions
log.Printf("[INFO] Should have deleted workflow %s", fileId)
err = DeleteKey(ctx, "workflow", fileId)
err = shuffle.DeleteKey(ctx, "workflow", fileId)
if err != nil {
log.Printf("Failed deleting key %s", fileId)
resp.WriteHeader(401)
@@ -2511,7 +2510,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) {
return
} else {
log.Printf("[INFO] Successfully ran cloud action STOP schedule")
err = DeleteKey(ctx, "schedules", scheduleId)
err = shuffle.DeleteKey(ctx, "schedules", scheduleId)
if err != nil {
log.Printf("[WARNING] Failed deleting cloud schedule onprem..")
resp.WriteHeader(401)
@@ -2634,7 +2633,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) {
func deleteSchedule(ctx context.Context, id string) error {
log.Printf("Should stop schedule %s!", id)
err := DeleteKey(ctx, "schedules", id)
err := shuffle.DeleteKey(ctx, "schedules", id)
if err != nil {
log.Printf("Failed to delete schedule: %s", err)
return err
@@ -3054,7 +3053,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
}
log.Printf("[INFO] Deleting public app")
err = DeleteKey(ctx, "workflowapp", fileId)
err = shuffle.DeleteKey(ctx, "workflowapp", fileId)
if err != nil {
log.Printf("Failed deleting workflowapp")
resp.WriteHeader(401)
@@ -4247,7 +4246,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
if len(removeApps) > 0 {
for _, item := range removeApps {
log.Printf("[WARNING] Removing duplicate app: %s", item)
err = DeleteKey(ctx, "workflowapp", item)
err = shuffle.DeleteKey(ctx, "workflowapp", item)
if err != nil {
log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err)
}
@@ -4950,3 +4949,20 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId
return nil
}
// Index = Username
func DeleteKeys(ctx context.Context, entity string, value []string) error {
// Non indexed User data
keys := []*datastore.Key{}
for _, item := range value {
keys = append(keys, datastore.NameKey(entity, item, nil))
}
err := dbclient.DeleteMulti(ctx, keys)
if err != nil {
log.Printf("Error deleting %s from %s: %s", value, entity, err)
return err
}
return nil
}
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.80
VERSION=0.8.81
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+2 -2
View File
@@ -244,11 +244,11 @@ func initializeImages() {
ctx := context.Background()
if appSdkVersion == "" {
appSdkVersion = "0.8.80"
appSdkVersion = "0.8.82"
log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
}
if workerVersion == "" {
workerVersion = "0.8.80"
workerVersion = "0.8.82"
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
functions/onprem/worker/Dockerfile
+16 -11
View File
@@ -1,20 +1,25 @@
FROM golang:1.16.0-buster as builder
WORKDIR /app
RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client
#RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client
#RUN go env -w GO111MODULE=auto
COPY worker.go /app/worker.go
RUN go mod init worker
RUN go get github.com/docker/docker/api/types && \
go get github.com/docker/docker/api/types/container && \
go get github.com/docker/docker/client && \
go get github.com/gorilla/mux && \
go get github.com/patrickmn/go-cache && \
go get github.com/frikky/shuffle-shared && \
go get github.com/satori/go.uuid && \
go get github.com/fsouza/go-dockerclient && \
go get google.golang.org/grpc/balancer/grpclb@v1.37.1
COPY go.mod /app/go.mod
COPY go.sum /app/go.sum
RUN go get
#RUN go mod init worker
#RUN go get
#RUN go get github.com/docker/docker/api/types && \
# go get github.com/docker/docker/api/types/container && \
# go get github.com/docker/docker/client && \
# go get github.com/gorilla/mux && \
# go get github.com/patrickmn/go-cache && \
# go get github.com/frikky/shuffle-shared && \
# go get github.com/satori/go.uuid && \
# go get github.com/fsouza/go-dockerclient && \
# go get google.golang.org/grpc/balancer/grpclb@v1.37.1
RUN go build
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
functions/onprem/worker/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.8.80
VERSION=0.8.82
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
functions/onprem/worker/go.mod
+1 -1
View File
@@ -12,7 +12,7 @@ require (
github.com/docker/docker v20.10.5+incompatible
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/frikky/shuffle-shared v0.0.40
github.com/frikky/shuffle-shared v0.0.45
github.com/fsouza/go-dockerclient v1.7.2
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
functions/onprem/worker/worker.go
+59 -54
View File
@@ -133,7 +133,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv)
}
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
abortUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
path := fmt.Sprintf("?reason=%s", url.QueryEscape(reason))
if len(nodeId) > 0 {
@@ -144,12 +144,12 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
}
//fmt.Println(url.QueryEscape(query))
fullUrl += path
log.Printf("[INFO] Abort URL: %s", fullUrl)
abortUrl += path
log.Printf("[INFO] Abort URL: %s", abortUrl)
req, err := http.NewRequest(
"GET",
fullUrl,
abortUrl,
nil,
)
@@ -426,15 +426,15 @@ func handleSubworkflowExecution(client *http.Client, workflowExecution shuffle.W
baseResult = `{"success": false}`
} else {
log.Printf("Should execute workflow %s with APIKEY %s and data %s", workflowId, apikey, executionArgument)
fullUrl := fmt.Sprintf("%s/api/workflows/%s/execute", baseUrl, workflowId)
executeUrl := fmt.Sprintf("%s/api/workflows/%s/execute", baseUrl, workflowId)
req, err := http.NewRequest(
"POST",
fullUrl,
executeUrl,
bytes.NewBuffer([]byte(executionArgument)),
)
if err != nil {
log.Printf("Error building test request: %s", err)
log.Printf("[WARNING] Error building test request: %s", err)
return err
}
@@ -477,10 +477,10 @@ func handleSubworkflowExecution(client *http.Client, workflowExecution shuffle.W
return err
}
fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
streamUrl,
bytes.NewBuffer([]byte(resultData)),
)
@@ -801,8 +801,10 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
// Execute, as we don't really care if env is not set? IDK
if action.Environment != environment { //&& action.Environment != "" {
//log.Printf("Action: %#v", action)
log.Printf("Bad environment for node: %s. Want %s", action.Environment, environment)
continue
log.Printf("[WARNING] Bad environment for node: %#v. Want %s. Skipping if NOT empty env.", action.Environment, environment)
if len(action.Environment) > 0 {
continue
}
}
// check whether the parent is finished executing
@@ -975,41 +977,47 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.")
image = images[2]
reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
buildBuf := new(strings.Builder)
_, err = io.Copy(buildBuf, reader)
if err != nil && !strings.Contains(fmt.Sprintf("Docker error: %s", err.Error()), "Conflict. The container name") {
log.Printf("[ERROR] Error in IO copy: %s", err)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
} else {
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
log.Printf("[INFO] Successfully downloaded %s", image)
}
err = deployApp(dockercli, image, identifier, env, workflowExecution)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist")
if strings.Contains(err.Error(), "exited prematurely") {
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
if strings.Contains(err.Error(), "No such image") {
//log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err)
log.Printf("[ERROR] Image doesn't exist. Shutting down")
log.Printf("[WARNING] Failed CLEANUP execution. Downloading image remotely.")
reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting %s. Couldn't be find locally, AND is missing.", image)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
buildBuf := new(strings.Builder)
_, err = io.Copy(buildBuf, reader)
if err != nil && !strings.Contains(fmt.Sprintf("Docker error: %s", err.Error()), "Conflict. The container name") {
log.Printf("[ERROR] Error in IO copy: %s", err)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
} else {
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image)
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
log.Printf("[INFO] Successfully downloaded %s", image)
}
err = deployApp(dockercli, image, identifier, env, workflowExecution)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist")
if strings.Contains(err.Error(), "exited prematurely") {
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
if strings.Contains(err.Error(), "No such image") {
//log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err)
log.Printf("[ERROR] Image doesn't exist. Shutting down")
shutdown(workflowExecution, action.ID, fmt.Sprintf("Docker error: %s", err.Error()), true)
}
}
}
}
} else {
@@ -1253,13 +1261,13 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec
ctx := context.Background()
setWorkflowExecution(ctx, workflowExecution, false)
streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl)
for {
//fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl)
//log.Printf("[INFO] URL: %s", fullUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
streamResultUrl,
bytes.NewBuffer([]byte(data)),
)
@@ -1375,10 +1383,10 @@ func runUserInput(client *http.Client, action shuffle.Action, workflowId, workfl
return err
}
fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
streamUrl,
bytes.NewBuffer([]byte(resultData)),
)
@@ -1404,10 +1412,10 @@ func runUserInput(client *http.Client, action shuffle.Action, workflowId, workfl
}
func runTestExecution(client *http.Client, workflowId, apikey string) (string, string) {
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", baseUrl, workflowId)
executeUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", baseUrl, workflowId)
req, err := http.NewRequest(
"GET",
fullUrl,
executeUrl,
nil,
)
@@ -1484,12 +1492,10 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
if workflowExecution.Status == "FINISHED" {
log.Printf("Workflowexecution is already FINISHED. No further action can be taken")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status)))
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because it has status %s"}`, workflowExecution.LastNode, workflowExecution.Status)))
return
}
// Not sure what's up here
// FIXME - remove comment
if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" {
if workflowExecution.Workflow.Configuration.ExitOnError {
@@ -1602,10 +1608,10 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) {
return
}
fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
streamUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
streamUrl,
bytes.NewBuffer([]byte(data)),
)
@@ -1777,11 +1783,11 @@ func runWebserver(listener net.Listener) {
func downloadDockerImage(client *http.Client, imageName string) {
data := fmt.Sprintf(`{"name": "%s"}`, imageName)
fullUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl)
dockerImgUrl := fmt.Sprintf("%s/api/v1/get_docker_image", baseUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
dockerImgUrl,
bytes.NewBuffer([]byte(data)),
)
@@ -1862,7 +1868,6 @@ func downloadDockerImage(client *http.Client, imageName string) {
// Initial loop etc
func main() {
log.Printf("[INFO] Setting up worker environment")
sleepTime := 5
@@ -1920,10 +1925,10 @@ func main() {
}
data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization)
fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl)
streamResultUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
streamResultUrl,
bytes.NewBuffer([]byte(data)),
)