From 2a825dd1ea66e7482a68b1d837eccb57773a2d26 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 11 Mar 2021 16:50:58 +0100 Subject: [PATCH] #293: Entire fix in a single button --- backend/app_sdk/app_base.py | 171 ++++++++++++++++- backend/app_sdk/build.sh | 2 +- backend/go-app/main.go | 245 +++++++++++++++++++++++++ backend/go-app/walkoff.go | 31 ++-- frontend/src/views/AngularWorkflow.jsx | 83 +++++++-- frontend/src/views/Workflows.jsx | 3 + functions/onprem/orborus/Dockerfile | 1 - functions/onprem/orborus/go.mod | 3 +- functions/onprem/orborus/go.sum | 7 + functions/onprem/worker/Dockerfile | 17 +- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 83 ++++++++- 12 files changed, 594 insertions(+), 54 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 1cb3dc80..c369533b 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -9,6 +9,7 @@ import requests import urllib.parse import http.client import urllib3 +import hashlib class AppBase: __version__ = None @@ -91,6 +92,135 @@ class AppBase: else: return {()} + # Handles unique fields by negoiating with the backend + def validate_unique_fields(self, params): + #print("IN THE UNIQUE FIELDS PLACE!") + + newlist = [params] + if isinstance(params, list): + #print("ITS A LIST!") + newlist = params + + #self.full_execution = os.getenv("FULL_EXECUTION", "") + #print(len(params)) + #print(params.items()) + #print(list(params.items())) + #print(f"PARAM: {params}") + #print(f"NEWLIST: {newlist}") + + # FIXME: Also handle MULTI PARAM + values = [] + param_names = [] + all_values = {} + index = 0 + for outerparam in newlist: + + #print(f"INNERTYPE: {type(outerparam)}") + #print(f"HANDLING PARAM {key}") + param_value = "" + for key, value in outerparam.items(): + #print("KEY: %s" % key) + #value = params[key] + for param in self.action["parameters"]: + try: + if param["name"] == key and param["unique_toggled"]: + print(f"FOUND: {key} with param {param}!") + if isinstance(value, dict) or isinstance(value, list): + try: + value = json.dumps(value) + except json.decoder.JSONDecodeError as e: + print(f"Error in json decode for param {value}: {e}") + continue + elif isinstance(value, int) or isinstance(value, float): + value = str(value) + elif value == False: + value = "False" + elif value == True: + value = "True" + + print(f"VALUE APPEND: {value}") + param_value += value + + if param["name"] not in param_names: + param_names.append(param["name"]) + except (KeyError, NameError) as e: + print(f"Key/NameError in param handler: {e}") + + print(f"OUTER VALUE: {param_value}") + if len(param_value) > 0: + md5 = hashlib.md5(param_value.encode('utf-8')).hexdigest() + values.append(md5) + all_values[md5] = { + "index": index, + } + + index += 1 + + # When in here, it means it should be unique + # Should this be done by the backend? E.g. ask it if the value is valid? + # 1. Check if it's unique towards key:value store in org for action + # 2. Check if COMBINATION is unique towards key:value store of action for org + # 3. Have a workflow configuration for unique ID's in unison or per field? E.g. if toggled, then send a hash of all fields together alphabetically, but if not, send one field at a time + + # org_id = full_execution["workflow"]["execution_org"]["id"] + + # USE ARRAY? + + new_params = [] + if len(values) > 0: + org_id = self.full_execution["workflow"]["execution_org"]["id"] + data = { + "append": True, + "workflow_check": False, + "authorization": self.authorization, + "execution_ref": self.current_execution_id, + "org_id": org_id, + "values": [{ + "app": self.action["app_name"], + "action": self.action["name"], + "parameternames": param_names, + "parametervalues": values, + }] + } + + #print(f"DATA: {data}") + # 1594869a676630b397bc34f7dc0951a3 + + #print(f"VALUE URL: {url}") + #print(f"RET: {ret.text}") + #print(f"ID: {ret.status_code}") + url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" + ret = requests.post(url, json=data) + if ret.status_code == 200: + json_value = ret.json() + if len(json_value["found"]) > 0: + modifier = 0 + for item in json_value["found"]: + print(f"Should remove {item}") + + try: + print(f"FOUND: {all_values[item]}") + print(f"SHOULD REMOVE INDEX: {all_values[item]['index']}") + + try: + newlist.pop(all_values[item]["index"]-modifier) + modifier += 1 + except IndexError as e: + print(f"Error popping value from array: {e}") + except (NameError, KeyError) as e: + print(f"Failed removal: {e}") + + + #return False + else: + print("None of the items were found!") + return newlist + else: + print(f"[WARNING] Failed checking values with status code {ret.status_code}!") + + #return True + return newlist + # Returns a list of all the executions to be done in the inner loop # FIXME: Doesn't take into account whether you actually WANT to loop or not # Check if the last part of the value is #? @@ -347,6 +477,26 @@ class AppBase: ret = [] param_multiplier = await self.get_param_multipliers(newparams) + # FIXME: This does a deduplication of the data + new_params = self.validate_unique_fields(param_multiplier) + print(f"NEW PARAMS: {new_params}") + if len(new_params) == 0: + print(f"No ID's to handle for validation") + else: + #subparams = new_params + print(f"NEW PARAMS: {new_params}") + + #print("Returned with newparams of length %d", len(new_params)) + #if isinstance(new_params, list) and len(new_params) == 1: + # params = new_params[0] + #else: + # print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + # action_result["status"] = "SKIPPED" + # action_result["result"] = f"A non-unique value was found" + # action_result["completed_at"] = int(time.time()) + # self.send_result(action_result, headers, stream_path) + # return + print("[INFO] Multiplier length: %d" % len(param_multiplier)) for subparams in param_multiplier: print(f"SUBPARAMS IN MULTI: {subparams}") @@ -1398,6 +1548,7 @@ class AppBase: return True, "" + # THE START IS ACTUALLY RIGHT HERE :O # Checks whether conditions are met, otherwise set branchcheck, tmpresult = check_branch_conditions(action, fullexecution) @@ -1848,14 +1999,19 @@ class AppBase: #print() if not multiexecution: - #newparams.append({ - # "name": val["key"], - # "value": val["value"], - # "variant": "STATIC_VALUE", - # "id": "body_replacement", - #}) + # Runs a single iteration here + new_params = self.validate_unique_fields(params) + print(f"Returned with newparams of length {len(new_params)}") + if isinstance(new_params, list) and len(new_params) == 1: + params = new_params[0] + else: + print("[WARNING] SHOULD STOP EXECUTION BECAUSE FIELDS AREN'T UNIQUE") + action_result["status"] = "SKIPPED" + action_result["result"] = f"A non-unique value was found" + action_result["completed_at"] = int(time.time()) + self.send_result(action_result, headers, stream_path) + return - #print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") print("[INFO] Running normal execution\n") newres = await func(**params) print("\n[INFO] Returned from execution!")#, newres) @@ -2058,7 +2214,6 @@ class AppBase: print("Normal result - no list?") result = results - print("RESULT: %s" % result) action_result["status"] = "SUCCESS" action_result["result"] = str(result) if action_result["result"] == "": diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index ffed66c5..ae25a4a3 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.61 +VERSION=0.8.62 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index cc620eed..57d57e75 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -7722,6 +7722,247 @@ func handleStopCloudSync(syncUrl string, org Org) error { return nil } +func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + body, err := ioutil.ReadAll(request.Body) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) + return + } + + // Append: Checks if the value should be appended + // WorkflowCheck: Checks if the value should only check for workflow in org, or entire org + // Authorization: the Authorization to use + // ExecutionRef: Ref for the execution + // Values: The values to use + type DataValues struct { + App string + Actions string + ParameterNames []string + ParameterValues []string + } + + type ReturnData struct { + Append bool `json:"append"` + WorkflowCheck bool `json:"workflow_check"` + Authorization string `json:"authorization"` + ExecutionRef string `json:"execution_ref"` + OrgId string `json:"org_id"` + Values []DataValues `json:"values"` + } + + //for key, value := range data.Apps { + var fileId string + location := strings.Split(request.URL.String(), "/") + if location[1] == "api" { + if len(location) <= 4 { + log.Printf("Path too short: %d", len(location)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + fileId = location[4] + } + + var tmpData ReturnData + err = json.Unmarshal(body, &tmpData) + if err != nil { + log.Printf("Failed unmarshalling test: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if tmpData.OrgId != fileId { + log.Printf("[INFO] OrgId %s and %s don't match", tmpData.OrgId, fileId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Organization ID's don't match"}`)) + return + } + + ctx := context.Background() + + org, err := getOrg(ctx, tmpData.OrgId) + if err != nil { + log.Printf("[INFO] Organization doesn't exist: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + workflowExecution, err := getWorkflowExecution(ctx, tmpData.ExecutionRef) + if err != nil { + log.Printf("[INFO] User can't edit the org") + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "No permission to get execution"}`)) + return + } + + if workflowExecution.Authorization != tmpData.Authorization { + log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Auth doesn't match"}`)) + return + } + + if workflowExecution.Status != "EXECUTING" { + log.Printf("[INFO] Workflow isn't executing and shouldn't be searching", workflowExecution.ExecutionId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Workflow isn't executing"}`)) + return + } + + if workflowExecution.ExecutionOrg != org.Id { + log.Printf("[INFO] Org %s wasn't used to execute %s", org.Id, workflowExecution.ExecutionId) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Bad organization specified"}`)) + return + } + + // Prepared for the future~ + if len(tmpData.Values) != 1 { + log.Printf("[INFO] Filter data can only hande 1 value right now, not %d", len(tmpData.Values)) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "Can't handle multiple apps yet, just one"}`)) + return + } + + value := tmpData.Values[0] + + // FIXME: Alphabetically sort the parameternames + // FIXME: Add organization wide search, not just workflow based + + found := []string{} + notFound := []string{} + + dbKey := fmt.Sprintf("app_execution_values") + parameterNames := fmt.Sprintf("%s_%s", value.App, strings.Join(value.ParameterNames, "_")) + log.Printf("[INFO] PARAMNAME: %s", parameterNames) + if tmpData.WorkflowCheck { + //for _, item := range tmpData.Values { + // log.Printf("[INFO] Should validate if values %#v in app parameter %#v exists WITH WORKFLOW %s", item.ParameterValues, item.ParameterNames, workflowExecution.Workflow.ID) + + // FIXME: Make this alphabetical + + for _, value := range value.ParameterValues { + if len(value) == 0 { + log.Printf("Shouldn't have value of length 0!") + continue + } + + log.Printf("[INFO] Looking for value %s", value) + + q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", workflowExecution.Workflow.ID).Filter("parameter_name =", parameterNames).Filter("value =", value) + foundCount, err := dbclient.Count(ctx, q) + if err != nil { + log.Printf("[WARNING] Failed getting key %s: %s", dbKey, err) + notFound = append(notFound, value) + //found = append(found, value) + continue + } + + if foundCount > 0 { + found = append(found, value) + } else { + log.Printf("[INFO] Found for %s: %d", dbKey, foundCount) + notFound = append(notFound, value) + } + } + } else { + log.Printf("[INFO] Should validate if value %s in app %s exists WITH ORG %s", workflowExecution.Workflow.ID) + + for _, value := range value.ParameterValues { + if len(value) == 0 { + log.Printf("Shouldn't have value of length 0!") + continue + } + + log.Printf("[INFO] Looking for value %s", value) + + q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id =", "").Filter("parameter_name =", parameterNames).Filter("value =", value) + foundCount, err := dbclient.Count(ctx, q) + if err != nil { + log.Printf("[WARNING] Failed getting key %s: %s", dbKey, err) + notFound = append(notFound, value) + //found = append(found, value) + continue + } + + if foundCount > 0 { + found = append(found, value) + } else { + log.Printf("[INFO] Found for %s: %d", dbKey, foundCount) + notFound = append(notFound, value) + } + } + } + + //App string + //Actions string + //ParameterNames string + //ParamererValues []string + + appended := 0 + if tmpData.Append { + log.Printf("[INFO] Should append %d values!", len(notFound)) + dbKey := fmt.Sprintf("app_execution_values") + + //q := datastore.NewQuery(dbKey).Filter("org_id =", org.Id).Filter("workflow_id", workflowExecution.Workflow.ID).Filter("app_name =", parameterNames).Filter("value =", value) + key := datastore.NameKey(dbKey, "", nil) + type NewValue struct { + OrgId string `json:"org_id" datastore:"org_id"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + WorkflowExecutionId string `json:"workflow_execution_id" datastore:"workflow_execution_id"` + ParameterName string `json:"parameter_name" datastore:"parameter_name"` + Value string `json:"value" datastore:"value"` + } + + //parameterNames := strings.Join(value.ParameterNames, "_") + for _, notFoundValue := range notFound { + newRequest := NewValue{ + OrgId: org.Id, + WorkflowExecutionId: workflowExecution.ExecutionId, + ParameterName: parameterNames, + Value: notFoundValue, + } + + if tmpData.WorkflowCheck { + newRequest.WorkflowId = workflowExecution.Workflow.ID + } + + if _, err := dbclient.Put(ctx, key, &newRequest); err != nil { + log.Printf("Error adding %s to appvalue: %s", notFoundValue, err) + continue + } + + appended += 1 + log.Printf("[INFO] Added %s as new appvalue to datastore", notFoundValue) + } + } + + type returnStruct struct { + Success bool `json:"success"` + Appended int `json:"appended"` + Found []string `json:"found"` + } + + returnData := returnStruct{ + Success: true, + Appended: appended, + Found: found, + } + + b, _ := json.Marshal(returnData) + resp.WriteHeader(200) + resp.Write(b) +} + func handleEditOrg(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -8432,6 +8673,10 @@ func initHandlers() { r.HandleFunc("/api/v1/orgs/", handleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", handleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") + + // This is a new API that validates if a key has been seen before. + // Not sure what the best course of action is for it. + r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", handleKeyValueCheck).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/orgs/{orgId}", handleEditOrg).Methods("POST", "OPTIONS") // Docker orborus specific diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 8f28f72e..05990f8b 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -229,6 +229,7 @@ type WorkflowAppActionParameter struct { Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` + UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` } type Valuereplace struct { @@ -1187,6 +1188,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Find underlying nodes and add them } else { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) + // Finds ALL childnodes to set them to SKIPPED childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) @@ -6119,8 +6121,6 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Starting app hotloading") - cacheKey := fmt.Sprintf("workflowapps-sorted-100") requestCache.Delete(cacheKey) cacheKey = fmt.Sprintf("workflowapps-sorted-500") @@ -6149,7 +6149,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("[INFO] Hotloading from %s", location) + log.Printf("[INFO] Starting hotloading from %s", location) err = handleAppHotload(location, true) if err != nil { log.Printf("Failed app hotload: %s", err) @@ -6251,6 +6251,20 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { } else { log.Printf("Updating apps with updates") } + + if tmpBody.ForceUpdate { + ctx := context.Background() + dockercli, err := client.NewEnvClient() + if err == nil { + _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) + if err != nil { + log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) + } + } else { + log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) + } + } + iterateAppGithubFolders(fs, dir, "", "", tmpBody.ForceUpdate) } else if strings.Contains(tmpBody.URL, "s3") { @@ -6568,17 +6582,6 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin buildLaterList := []buildLaterStruct{} ctx := context.Background() - if forceUpdate { - dockercli, err := client.NewEnvClient() - if err == nil { - _, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{}) - if err != nil { - log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err) - } - } else { - log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err) - } - } // It's here to prevent getting them in every iteration for _, file := range dir { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 3485f90c..45f44ba9 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1,5 +1,6 @@ import React, {useState, useEffect, useLayoutEffect} from 'react'; import { useInterval } from 'react-powerhooks'; +import { useTheme } from '@material-ui/core/styles'; import uuid from "uuid"; import {Link} from 'react-router-dom'; @@ -26,6 +27,7 @@ import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; import { validateJson } from "./Workflows.jsx"; import { GetParsedPaths } from "./Apps.jsx"; +import ConfigureWorkflow from '../components/ConfigureWorkflow.jsx'; const surfaceColor = "#27292D" const inputColor = "#383B40" @@ -87,6 +89,7 @@ const AngularWorkflow = (props) => { const referenceUrl = globalUrl+"/api/v1/hooks/" const alert = useAlert() const borderRadius = 3 + const theme = useTheme(); const [bodyWidth, bodyHeight] = useWindowSize(); const appBarSize = 74 @@ -126,6 +129,7 @@ const AngularWorkflow = (props) => { const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) const [lastExecution, setLastExecution] = React.useState("") + const [configureWorkflowModalOpen, setConfigureWorkflowModalOpen] = React.useState(false) const [curpath, setCurpath] = useState(typeof window === 'undefined' || window.location === undefined ? "" : window.location.pathname) // 0 = normal, 1 = just done, 2 = normal @@ -639,7 +643,7 @@ const AngularWorkflow = (props) => { getWorkflowExecution(props.match.params.key, "") } else if (responseJson.status === "FINISHED") { - console.log("STOPPING BECAUSE ITS OVAH!") + //console.log("STOPPING BECAUSE ITS OVAH!") setExecutionRunning(false) stop() getWorkflowExecution(props.match.params.key, "") @@ -1403,7 +1407,7 @@ const AngularWorkflow = (props) => { // might just be confusing cy.nodes().some(function( ele ) { if (ele.id() !== workflow.start && ele.data()["label"] !== undefined) { - alert.success("Changed startnode to "+ele.data()["label"]) + //alert.success("Changed startnode to "+ele.data()["label"]) ele.data("isStartNode", true) workflow.start = ele.id() //throw BreakException @@ -3734,7 +3738,7 @@ const AngularWorkflow = (props) => { return (
-
+
{data.configuration === true ? @@ -3744,15 +3748,15 @@ const AngularWorkflow = (props) => { }}/> : -
+
} -
+
{tmpitem}
- {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : + {/*selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null :
{ @@ -3781,6 +3785,29 @@ const AngularWorkflow = (props) => {
+ */} + {selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 && selectedActionParameters[count].required === true && selectedActionParameters[count].unique_toggled !== undefined ? null : +
+ +
{}}> + { + //console.log("CHECKED!: ", selectedActionParameters[count]) + selectedActionParameters[count].unique_toggled = !selectedActionParameters[count].unique_toggled + selectedAction.parameters[count].unique_toggled = selectedActionParameters[count].unique_toggled + setSelectedActionParameters(selectedActionParameters) + setSelectedAction(selectedAction) + setUpdate(Math.random()) + }} + name="requires_unique" + /> +
+ +
}
{datafield} @@ -4015,14 +4042,13 @@ const AngularWorkflow = (props) => { /> {selectedApp.name !== undefined && selectedAction.authentication !== undefined && selectedAction.authentication.length === 0 && requiresAuthentication ?
- Authenticate {selectedApp.name}: - +
@@ -6117,26 +6143,26 @@ const AngularWorkflow = (props) => { const topBarStyle= { position: "fixed", right: 0, - left: leftBarSize, - top: appBarSize, + left: leftBarSize+20, + top: appBarSize+20, + /* minWidth: cytoscapeViewWidths, maxWidth: cytoscapeViewWidths, - marginLeft: 20, - marginBottom: 20, + */ } const TopCytoscapeBar = () => { return (
-
+
-

+

Workflows

-

+

{workflow.name}

@@ -7472,6 +7498,24 @@ const AngularWorkflow = (props) => { ) } + const configureWorkflowModal = configureWorkflowModalOpen && apps.length !== 0 ? + { + setConfigureWorkflowModalOpen(false) + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: 600, + padding: 15, + }, + }} + > + + + : null // This whole part is redundant. Made it part of Arguments instead. @@ -7502,6 +7546,7 @@ const AngularWorkflow = (props) => { {conditionsModal} {authenticationModal} {codePopoutModal} + {configureWorkflowModal} { const [editingWorkflow, setEditingWorkflow] = React.useState({}) const [executionLoading, setExecutionLoading] = React.useState(false) const [isDropzone, setIsDropzone] = React.useState(false); + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const { start, stop } = useInterval({ duration: 5000, @@ -1230,11 +1231,13 @@ const Workflows = (props) => { : null} + {isCloud ? null : + } const WorkflowView = () => { diff --git a/functions/onprem/orborus/Dockerfile b/functions/onprem/orborus/Dockerfile index 3fe760c7..7c178fd8 100644 --- a/functions/onprem/orborus/Dockerfile +++ b/functions/onprem/orborus/Dockerfile @@ -2,7 +2,6 @@ FROM golang:1.16.0-buster as builder RUN mkdir /app WORKDIR /app -RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client COPY orborus.go /app/orborus.go RUN go mod init orborus diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod index 47154cda..4e17a529 100644 --- a/functions/onprem/orborus/go.mod +++ b/functions/onprem/orborus/go.mod @@ -3,13 +3,14 @@ module orborus go 1.13 require ( + github.com/Microsoft/go-winio v0.4.16 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.1+incompatible github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/gogo/protobuf v1.3.1 // indirect - github.com/mackerelio/go-osstat v0.1.0 // indirect + github.com/mackerelio/go-osstat v0.1.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum index 84c5d0ec..e8c720de 100644 --- a/functions/onprem/orborus/go.sum +++ b/functions/onprem/orborus/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -41,6 +43,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/mackerelio/go-osstat v0.1.0 h1:e57QHeHob8kKJ5FhcXGdzx5O6Ktuc5RHMDIkeqhgkFA= github.com/mackerelio/go-osstat v0.1.0/go.mod h1:1K3NeYLhMHPvzUu+ePYXtoB58wkaRpxZsGClZBJyIFw= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -53,9 +56,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -73,8 +78,10 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190410235845-0ad05ae3009d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/functions/onprem/worker/Dockerfile b/functions/onprem/worker/Dockerfile index 3bf1541a..a9df5c2b 100644 --- a/functions/onprem/worker/Dockerfile +++ b/functions/onprem/worker/Dockerfile @@ -1,18 +1,21 @@ FROM golang:1.16.0-buster as builder WORKDIR /app +RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client +#RUN go env -w GO111MODULE=auto COPY worker.go /app/worker.go -RUN go env -w GO111MODULE=auto - -RUN go get github.com/docker/docker/api/types -RUN go get github.com/docker/docker/api/types/container -RUN go get github.com/docker/docker/client -RUN go get github.com/gorilla/mux -RUN go get github.com/patrickmn/go-cache +RUN go mod init worker +RUN go get github.com/docker/docker/api/types && \ + go get github.com/docker/docker/api/types/container && \ + go get github.com/docker/docker/client && \ + go get github.com/gorilla/mux && \ + go get github.com/patrickmn/go-cache +RUN go build RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker . +## ALPINE IMAGE FROM alpine:3.12 ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 3f2a483e..44d134c5 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.62 +VERSION=0.8.63 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 3e15c282..0594b142 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -524,7 +524,7 @@ type WorkflowAppActionParameter struct { Description string `json:"description" datastore:"description,noindex" yaml:"description"` ID string `json:"id" datastore:"id" yaml:"id,omitempty"` Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example" yaml:"example"` + Example string `json:"example" datastore:"example,noindex" yaml:"example"` Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` Options []string `json:"options" datastore:"options" yaml:"options"` @@ -536,6 +536,7 @@ type WorkflowAppActionParameter struct { Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` + UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` } type Valuereplace struct { @@ -2335,9 +2336,9 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } else { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Continuing anyway because of workflow configuration.", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) // Finds ALL childnodes to set them to SKIPPED - childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) // Remove duplicates //log.Printf("CHILD NODES: %d", len(childNodes)) + childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) for _, nodeId := range childNodes { if nodeId == actionResult.Action.ID { continue @@ -2490,6 +2491,84 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl log.Printf("[INFO] Setting value (2) of %s in execution %s to %s. New result length: %d", actionResult.Action.ID, workflowExecution.ExecutionId, actionResult.Status, len(workflowExecution.Results)) } + if actionResult.Status == "SKIPPED" { + log.Printf("\n\n[INFO] Handling special case for SKIPPED!\n\n") + childNodes := findChildNodes(*workflowExecution, actionResult.Action.ID) + for _, nodeId := range childNodes { + if nodeId == actionResult.Action.ID { + continue + } + + // 1. Find the action itself + // 2. Create an actionresult + curAction := Action{ID: ""} + for _, action := range workflowExecution.Workflow.Actions { + if action.ID == nodeId { + curAction = action + break + } + } + + if len(curAction.ID) == 0 { + log.Printf("Couldn't find subnode %s", nodeId) + continue + } + + resultExists := false + for _, result := range workflowExecution.Results { + if result.Action.ID == curAction.ID { + resultExists = true + break + } + } + + if !resultExists { + // Check parents are done here. Only add it IF all parents are skipped + skipNodeAdd := false + for _, branch := range workflowExecution.Workflow.Branches { + if branch.DestinationID == nodeId { + // If the branch's source node is NOT in childNodes, it's not a skipped parent + sourceNodeFound := false + for _, item := range childNodes { + if item == branch.SourceID { + sourceNodeFound = true + break + } + } + + if !sourceNodeFound { + log.Printf("[INFO] Not setting node %s to SKIPPED", nodeId) + skipNodeAdd = true + break + } + } + } + + if !skipNodeAdd { + newAction := Action{ + AppName: curAction.AppName, + AppVersion: curAction.AppVersion, + Label: curAction.Label, + Name: curAction.Name, + ID: curAction.ID, + } + + newResult := ActionResult{ + Action: newAction, + ExecutionId: actionResult.ExecutionId, + Authorization: actionResult.Authorization, + Result: "Skipped because of previous node", + StartedAt: 0, + CompletedAt: 0, + Status: "SKIPPED", + } + + workflowExecution.Results = append(workflowExecution.Results, newResult) + } + } + } + } + // FIXME: Have a check for skippednodes and their parents /* for resultIndex, result := range workflowExecution.Results {