marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #103 from frikky/1.0.0

Browse Source 
0.6.0
This commit is contained in:
Frikky
2020-07-17 17:56:33 +09:00
committed by GitHub
parent 4b51175058 263bbbd987
commit 04c626cfb5
18 changed files with 2129 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/app_base.py
+86 -57
View File
@@ -21,25 +21,36 @@ class AppBase:
# authorization is for the specific workflow
self.url = os.getenv("CALLBACK_URL", "https://shuffler.io")
self.action = os.getenv("ACTION", "")
self.apikey = os.getenv("FUNCTION_APIKEY", "")
self.authorization = os.getenv("AUTHORIZATION", "")
self.current_execution_id = os.getenv("EXECUTIONID", "")
if len(self.action) == 0:
print("ACTION env not defined")
sys.exit(0)
if len(self.apikey) == 0:
print("FUNCTION_APIKEY env not defined")
sys.exit(0)
if len(self.authorization) == 0:
print("AUTHORIZATION env not defined")
sys.exit(0)
if len(self.current_execution_id) == 0:
print("EXECUTIONID env not defined")
sys.exit(0)
self.full_execution = os.getenv("FULL_EXECUTION", "")
if isinstance(self.action, str):
self.action = json.loads(self.action)
def send_result(self, action_result, headers, stream_path):
if action_result["status"] == "EXECUTING":
action_result["status"] = "FAILURE"
# I wonder if this actually works
self.logger.info("Before last stream result")
try:
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Result: %d" % ret.status_code)
if ret.status_code != 200:
self.logger.info(ret.text)
except requests.exceptions.ConnectionError as e:
self.logger.exception(e)
return
except TypeError as e:
self.logger.exception(e)
action_result["status"] = "FAILURE"
action_result["result"] = "POST error: %s" % e
self.logger.info("Before typeerror stream result")
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Result: %d" % ret.status_code)
if ret.status_code != 200:
self.logger.info(ret.text)
async def execute_action(self, action):
# FIXME - add request for the function STARTING here. Use "results stream" or something
@@ -58,9 +69,25 @@ class AppBase:
}
self.logger.info("ACTION RESULT: %s", action_result)
if len(self.action) == 0:
print("ACTION env not defined")
action_result["result"] = "Error in setup ENV: ACTION not defined"
self.send_result(action_result, headers, stream_path)
return
if len(self.authorization) == 0:
print("AUTHORIZATION env not defined")
action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined"
self.send_result(action_result, headers, stream_path)
return
if len(self.current_execution_id) == 0:
print("EXECUTIONID env not defined")
action_result["result"] = "Error in setup ENV: EXECUTIONID not defined"
self.send_result(action_result, headers, stream_path)
return
headers = {
"Content-Type": "application/json",
"Authorization": "Bearer %s" % self.apikey
"Authorization": "Bearer %s" % self.authorization
}
# Add async logger
@@ -73,33 +100,53 @@ class AppBase:
self.logger.info(ret.text)
except requests.exceptions.ConnectionError as e:
print("Connectionerror: %s" % e)
action_result["result"] = "Bad setup during startup: %d" % e
self.send_result(action_result, headers, stream_path)
return
# Verify whether there are any parameters with ACTION_RESULT required
# If found, we get the full results list from backend
fullexecution = {}
try:
tmpdata = {
"authorization": self.authorization,
"execution_id": self.current_execution_id
}
if len(self.full_execution) == 0:
print("NO EXECUTION - LOADING!")
try:
tmpdata = {
"authorization": self.authorization,
"execution_id": self.current_execution_id
}
self.logger.info("Before FULLEXEC stream result")
ret = requests.post(
"%s/api/v1/streams/results" % (self.url),
headers=headers,
json=tmpdata
)
self.logger.info("Before FULLEXEC stream result")
ret = requests.post(
"%s/api/v1/streams/results" % (self.url),
headers=headers,
json=tmpdata
)
if ret.status_code == 200:
fullexecution = ret.json()
else:
self.logger.info("Error: Data: ", ret.json())
self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code)
if ret.status_code == 200:
fullexecution = ret.json()
else:
self.logger.info("Error: Data: ", ret.json())
self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code)
action_result["result"] = "Bad result from backend: %d" % ret.status_code
self.send_result(action_result, headers, stream_path)
return
except requests.exceptions.ConnectionError as e:
self.logger.info("Connectionerror: %s" % e)
action_result["result"] = "Connection error during startup: %s" % e
self.send_result(action_result, headers, stream_path)
return
except requests.exceptions.ConnectionError as e:
self.logger.info("Connectionerror: %s" % e)
return
else:
try:
fullexecution = json.loads(self.full_execution)
except json.decoder.JSONDecodeError as e:
print("Json decode execution error: %s" % e)
action_result["result"] = "Json error during startup: %s" % e
self.send_result(action_result, headers, stream_path)
return
print("")
self.logger.info("AFTER FULLEXEC stream result")
@@ -810,11 +857,11 @@ class AppBase:
result += newres
else:
try:
result += str(result)
result += str(newres)
except ValueError:
result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
print("Can't handle type %s value from function" % (type(newres)))
print("POST NEWRES: ", newres)
print("POST NEWRES RESULT: ", result)
else:
print("APP_SDK DONE: Starting MULTI execution with", multi_parameters)
# 1. Use number of executions based on longest array
@@ -886,27 +933,9 @@ class AppBase:
action_result["completed_at"] = int(time.time())
# I wonder if this actually works
self.logger.info("Before last stream result")
try:
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Result: %d" % ret.status_code)
if ret.status_code != 200:
self.logger.info(ret.text)
except requests.exceptions.ConnectionError as e:
self.logger.exception(e)
return
except TypeError as e:
self.logger.exception(e)
action_result["status"] = "FAILURE"
action_result["result"] = "POST error: %s" % e
self.logger.info("Before typeerror stream result")
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
self.logger.info("Result: %d" % ret.status_code)
if ret.status_code != 200:
self.logger.info(ret.text)
return
# Send the result :)
self.send_result(action_result, headers, stream_path)
return
#STOPCOPY
backend/go-app/codegen.go
+88 -45
View File
@@ -398,8 +398,7 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
verifyAddin,
)
//log.Println(data)
//log.Println(functionname)
log.Printf("%s", data)
return functionname, data
}
@@ -459,73 +458,105 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
//log.Printf("%#v", securitySchemes)
api.Authentication = Authentication{
Required: true,
Parameters: []AuthenticationParams{
AuthenticationParams{
Multiline: false,
Required: true,
},
},
Required: true,
Parameters: []AuthenticationParams{},
}
// Used for python code generation lol
// Not sure how this should work with oauth
if securitySchemes["BearerAuth"] != nil {
api.Authentication.Parameters[0].Value = "BearerAuth"
api.Authentication.Parameters[0].Description = securitySchemes["BearerAuth"].Value.Description
api.Authentication.Parameters[0].Name = securitySchemes["BearerAuth"].Value.Name
api.Authentication.Parameters[0].In = securitySchemes["BearerAuth"].Value.In
api.Authentication.Parameters[0].Scheme = securitySchemes["BearerAuth"].Value.Scheme
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "apikey",
Value: "",
Example: "******",
Description: securitySchemes["BearerAuth"].Value.Description,
In: securitySchemes["BearerAuth"].Value.In,
Scheme: securitySchemes["BearerAuth"].Value.Scheme,
Schema: SchemaDefinition{
Type: securitySchemes["BearerAuth"].Value.Scheme,
},
})
//log.Printf("HANDLE BEARER AUTH")
extraParameters = append(extraParameters, WorkflowAppActionParameter{
Name: "apikey",
Description: "The apikey to use",
Multiline: false,
Required: true,
Example: "The API key to use. Space = skip",
Name: "apikey",
Description: "The apikey to use",
Multiline: false,
Required: true,
Example: "The API key to use. Space = skip",
Configuration: true,
Schema: SchemaDefinition{
Type: "string",
},
})
} else if securitySchemes["ApiKeyAuth"] != nil {
api.Authentication.Parameters[0].Value = "ApiKeyAuth"
api.Authentication.Parameters[0].Description = securitySchemes["ApiKeyAuth"].Value.Description
api.Authentication.Parameters[0].Name = securitySchemes["ApiKeyAuth"].Value.Name
api.Authentication.Parameters[0].In = securitySchemes["ApiKeyAuth"].Value.In
api.Authentication.Parameters[0].Scheme = securitySchemes["ApiKeyAuth"].Value.Scheme
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "apikey",
Value: "",
Example: "******",
Description: securitySchemes["ApiKeyAuth"].Value.Description,
In: securitySchemes["ApiKeyAuth"].Value.In,
Scheme: securitySchemes["ApiKeyAuth"].Value.Scheme,
Schema: SchemaDefinition{
Type: securitySchemes["ApiKeyAuth"].Value.Scheme,
},
})
//log.Printf("HANDLE APIKEY AUTH")
extraParameters = append(extraParameters, WorkflowAppActionParameter{
Name: "apikey",
Description: "The apikey to use",
Multiline: false,
Required: true,
Example: "**********",
Name: "apikey",
Description: "The apikey to use",
Multiline: false,
Required: true,
Example: "**********",
Configuration: true,
Schema: SchemaDefinition{
Type: "string",
},
})
} else if securitySchemes["BasicAuth"] != nil {
api.Authentication.Parameters[0].Value = "BasicAuth"
api.Authentication.Parameters[0].Description = securitySchemes["BasicAuth"].Value.Description
api.Authentication.Parameters[0].Name = securitySchemes["BasicAuth"].Value.Name
api.Authentication.Parameters[0].In = securitySchemes["BasicAuth"].Value.In
api.Authentication.Parameters[0].Scheme = securitySchemes["BasicAuth"].Value.Scheme
extraParameters = append(extraParameters, WorkflowAppActionParameter{
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "username",
Description: "The username to use",
Multiline: false,
Required: true,
Example: "The username to use",
Value: "",
Example: "username",
Description: securitySchemes["BasicAuth"].Value.Description,
In: securitySchemes["BasicAuth"].Value.In,
Scheme: securitySchemes["BasicAuth"].Value.Scheme,
Schema: SchemaDefinition{
Type: securitySchemes["BasicAuth"].Value.Scheme,
},
})
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "password",
Value: "",
Example: "*****",
Description: securitySchemes["BasicAuth"].Value.Description,
In: securitySchemes["BasicAuth"].Value.In,
Scheme: securitySchemes["BasicAuth"].Value.Scheme,
Schema: SchemaDefinition{
Type: securitySchemes["BasicAuth"].Value.Scheme,
},
})
extraParameters = append(extraParameters, WorkflowAppActionParameter{
Name: "username",
Description: "The username to use",
Multiline: false,
Required: true,
Example: "The username to use",
Configuration: true,
Schema: SchemaDefinition{
Type: "string",
},
})
extraParameters = append(extraParameters, WorkflowAppActionParameter{
Name: "password",
Description: "The password to use",
Multiline: false,
Required: true,
Example: "***********",
Name: "password",
Description: "The password to use",
Multiline: false,
Required: true,
Example: "***********",
Configuration: true,
Schema: SchemaDefinition{
Type: "string",
},
@@ -535,11 +566,23 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
// Adds a link parameter if it's not already defined
if len(api.Link) == 0 {
extraParameters = append(extraParameters, WorkflowAppActionParameter{
api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{
Name: "url",
Description: "The URL of the app",
Multiline: false,
Required: true,
Example: "https://shuffler.io",
Schema: SchemaDefinition{
Type: "string",
},
})
extraParameters = append(extraParameters, WorkflowAppActionParameter{
Name: "url",
Description: "The URL of the app",
Multiline: false,
Required: true,
Configuration: true,
Schema: SchemaDefinition{
Type: "string",
},
backend/go-app/main.go
+5 -1
View File
@@ -5954,7 +5954,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("Successfully uploaded ZIPFILE for %s", identifier)
log.Printf("Successfully stitched ZIPFILE for %s", identifier)
// 4. Upload as cloud function - this apikey is specifically for cloud functions rofl
//environmentVariables := map[string]string{
@@ -6452,6 +6452,10 @@ func init() {
r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/apps/search", getSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication", getAppAuthentication).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication", addAppAuthentication).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication/{appauthId}", deleteAppAuthentication).Methods("DELETE", "OPTIONS")
// Legacy app things
r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/apps", getWorkflowApps).Methods("GET", "OPTIONS")
backend/go-app/walkoff.go
+517 -28
View File
@@ -77,6 +77,23 @@ type Org struct {
Id string `json:"id"`
}
type AppAuthenticationStorage struct {
Active bool `json:"active" datastore:"active"`
Label string `json:"label" datastore:"label"`
Id string `json:"id" datastore:"id"`
App WorkflowApp `json:"app" datastore:"app"`
Fields []AuthenticationStore `json:"fields" datastore:"fields"`
Usage []AuthenticationUsage `json:"usage" datastore:"usage"`
WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"`
NodeCount int64 `json:"node_count" datastore:"node_count"`
}
type AuthenticationUsage struct {
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Nodes []string `json:"nodes" datastore:"nodes"`
}
// An app inside Shuffle
type WorkflowApp struct {
Name string `json:"name" yaml:"name" required:true datastore:"name"`
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
@@ -105,16 +122,17 @@ type WorkflowApp struct {
}
type WorkflowAppActionParameter struct {
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
Required bool `json:"required" datastore:"required" yaml:"required"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
Required bool `json:"required" datastore:"required" yaml:"required"`
Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
}
type SchemaDefinition struct {
@@ -142,9 +160,13 @@ type WorkflowAppAction struct {
} `json:"execution_variable" datastore:"execution_variables"`
Returns struct {
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
Example string `json:"example" datastore:"example" yaml:"example"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"returns" datastore:"returns"`
AuthenticationId string `json:"authentication_id" datastore:"authentication_id"`
Example string `json:"example" datastore:"example" yaml:"example"`
AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"`
}
// FIXME: Generate a callback authentication ID?
@@ -173,7 +195,7 @@ type WorkflowExecution struct {
} `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"`
}
// Added environment for location to execute
// This is for the nodes in a workflow, NOT the app action itself.
type Action struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
@@ -200,7 +222,10 @@ type Action struct {
X float64 `json:"x" datastore:"x"`
Y float64 `json:"y" datastore:"y"`
} `json:"position"`
Priority int `json:"priority" datastore:"priority"`
Priority int `json:"priority" datastore:"priority"`
AuthenticationId string `json:"authentication_id" datastore:"authentication_id"`
Example string `json:"example" datastore:"example"`
AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"`
}
// Added environment for location to execute
@@ -301,15 +326,16 @@ type Authentication struct {
}
type AuthenticationParams struct {
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
Required bool `json:"required" datastore:"required" yaml:"required"`
In string `json:"in" datastore:"in" yaml:"in"`
Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"`
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
Required bool `json:"required" datastore:"required" yaml:"required"`
In string `json:"in" datastore:"in" yaml:"in"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated
}
type AuthenticationStore struct {
@@ -596,6 +622,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
resp.Write([]byte("OK"))
}
// FIXME: Authenticate this one (especially since we have a default: shuffle)
func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -1375,6 +1402,60 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(`{"success": true}`))
}
// Adds app auth tracking
func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add bool) error {
workflowFound := false
workflowIndex := 0
nodeFound := false
for index, workflow := range auth.Usage {
if workflow.WorkflowId == workflowId {
// Check if node exists
workflowFound = true
workflowIndex = index
for _, actionId := range workflow.Nodes {
if actionId == nodeId {
nodeFound = true
break
}
}
break
}
}
// FIXME: Add a way to use !add to remove
updateAuth := false
if !workflowFound && add {
log.Printf("Adding workflow things to auth!")
usageItem := AuthenticationUsage{
WorkflowId: workflowId,
Nodes: []string{nodeId},
}
auth.Usage = append(auth.Usage, usageItem)
auth.WorkflowCount += 1
auth.NodeCount += 1
updateAuth = true
} else if !nodeFound && add {
log.Printf("Adding node things to auth!")
auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId)
auth.NodeCount += 1
updateAuth = true
}
if updateAuth {
log.Printf("Updating auth!")
ctx := context.Background()
err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id)
if err != nil {
log.Printf("Failed setting up app auth %s: %s", auth.Id, err)
return err
}
}
return nil
}
// Saves a workflow to an ID
func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
@@ -1470,7 +1551,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
// FIXME - this shouldn't be necessary with proper API checks
newActions := []Action{}
allNodes := []string{}
//log.Println("Pre")
//log.Printf("Action: %#v", action.Authentication)
for _, action := range workflow.Actions {
allNodes = append(allNodes, action.ID)
@@ -1615,6 +1697,14 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
}
allAuths, err := getAllWorkflowAppAuth(ctx)
if userErr != nil {
log.Printf("Api authentication failed in get all apps: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Check every app action and param to see whether they exist
newActions = []Action{}
for _, action := range workflow.Actions {
@@ -1632,6 +1722,30 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
}
// Check auth
// 1. Find the auth in question
// 2. Update the node and workflow info in the auth
// 3. Get the values in the auth and add them to the action values
if len(action.AuthenticationId) > 0 {
authFound := false
for _, auth := range allAuths {
if auth.Id == action.AuthenticationId {
authFound = true
// Updates the auth item itself IF necessary
go updateAppAuth(auth, workflow.ID, action.ID, true)
break
}
}
if !authFound {
log.Printf("App auth %s doesn't exist", action.AuthenticationId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App auth %s doesn't exist"}`, action.AuthenticationId)))
return
}
}
if builtin {
newActions = append(newActions, action)
} else {
@@ -1700,6 +1814,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
}
newParams = append(newParams, actionParam)
break
}
}
@@ -1925,7 +2040,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) {
var workflowExecutions []WorkflowExecution
_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
log.Printf("Error getting workflowexec: %s", err)
log.Printf("Error getting workflowexec (cleanup): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting all workflowexecutions"}`)))
return
@@ -2191,6 +2306,8 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
// FIXME - remove this?
newActions := []Action{}
defaultResults := []ActionResult{}
allAuths := []AppAuthenticationStorage{}
for _, action := range workflowExecution.Workflow.Actions {
action.LargeImage = ""
if action.ID == workflowExecution.Start {
@@ -2202,6 +2319,47 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
return WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!")
}
// FIXME: Authentication parameters
if len(action.AuthenticationId) > 0 {
if len(allAuths) == 0 {
allAuths, err = getAllWorkflowAppAuth(ctx)
if err != nil {
log.Printf("Api authentication failed in get all app auth: %s", err)
return WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err
}
}
curAuth := AppAuthenticationStorage{Id: ""}
for _, auth := range allAuths {
if auth.Id == action.AuthenticationId {
curAuth = auth
break
}
}
if len(curAuth.Id) == 0 {
return WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId))
}
// Rebuild params with the right data. This is to prevent issues on the frontend
newParams := []WorkflowAppActionParameter{}
for _, param := range action.Parameters {
for _, authparam := range curAuth.Fields {
if param.Name == authparam.Key {
log.Printf("Name: %s - value: %s", param.Name, param.Value)
param.Value = authparam.Value
log.Printf("Name: %s - value: %s\n", param.Name, param.Value)
break
}
}
newParams = append(newParams, param)
}
action.Parameters = newParams
}
newActions = append(newActions, action)
// If the node is NOT found, it's supposed to be set to SKIPPED,
@@ -2967,6 +3125,59 @@ func setWorkflow(ctx context.Context, workflow Workflow, id string) error {
return nil
}
func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
user, userErr := handleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("Api authentication failed in edit workflow: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("Need to be admin to delete appauth")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
log.Printf("%#v", location)
var fileId string
if location[1] == "api" {
if len(location) <= 5 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[5]
}
// FIXME: Set affected workflows to have errors
// 1. Get the auth
// 2. Loop the workflows (.Usage) and set them to have errors
// 3. Loop the nodes in workflows and do the same
log.Printf("ID: %s", fileId)
ctx := context.Background()
err := DeleteKey(ctx, "workflowappauth", fileId)
if err != nil {
log.Printf("Failed deleting workflowapp")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -3191,6 +3402,207 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) {
resp.Write(data)
}
func addAppAuthentication(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// FIXME - need to be logged in?
_, userErr := handleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("Api authentication failed in get all apps: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
var appAuth AppAuthenticationStorage
err = json.Unmarshal(body, &appAuth)
if err != nil {
log.Printf("Failed unmarshaling (appauth): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(appAuth.Id) == 0 {
appAuth.Id = uuid.NewV4().String()
}
ctx := context.Background()
if len(appAuth.Label) == 0 {
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Label can't be empty"}`)))
return
}
// Super basic check
if len(appAuth.App.ID) != 36 && len(appAuth.App.ID) != 32 {
log.Printf("Bad ID for app: %s", appAuth.App.ID)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App has to be defined"}`)))
return
}
app, err := getApp(ctx, appAuth.App.ID)
if err != nil {
log.Printf("Failed finding app %s while setting auth.", appAuth.App.ID)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
// Check if the items are correct
for _, field := range appAuth.Fields {
found := false
for _, param := range app.Authentication.Parameters {
if field.Key == param.Name {
found = true
}
}
if !found {
log.Printf("Failed finding field %s in appauth fields", field.Key)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "All auth fields required"}`)))
return
}
}
err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id)
if err != nil {
log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func getAppAuthentication(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
_, userErr := handleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("Api authentication failed in get all apps: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: Auth to get the right ones only
//if user.Role != "admin" {
// log.Printf("User isn't admin")
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false}`))
// return
//}
ctx := context.Background()
allAuths, err := getAllWorkflowAppAuth(ctx)
if err != nil {
log.Printf("Api authentication failed in get all app auth: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(allAuths) == 0 {
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "data": []}`))
return
}
// Cleanup for frontend
newAuth := []AppAuthenticationStorage{}
for _, auth := range allAuths {
newAuthField := auth
for index, _ := range auth.Fields {
newAuthField.Fields[index].Value = "auth placeholder (replaced during execution)"
}
newAuth = append(newAuth, newAuthField)
}
newbody, err := json.Marshal(allAuths)
if err != nil {
log.Printf("Failed unmarshalling all app auths: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow app auth"}`)))
return
}
data := fmt.Sprintf(`{"success": true, "data": %s}`, string(newbody))
resp.WriteHeader(200)
resp.Write([]byte(data))
/*
data := `{
"success": true,
"data": [
{
"app": {
"name": "thehive",
"description": "what",
"app_version": "1.0.0",
"id": "4f97da9d-1caf-41cc-aa13-67104d8d825c",
"large_image": "asd"
},
"fields": {
"apikey": "hello",
"url": "url"
},
"usage": [{
"workflow_id": "asd",
"nodes": [{
"node_id": ""
}]
}],
"label": "Original",
"id": "4f97da9d-1caf-41cc-aa13-67104d8d825d",
"active": true
},
{
"app": {
"name": "thehive",
"description": "what",
"app_version": "1.0.0",
"id": "4f97da9d-1caf-41cc-aa13-67104d8d825c",
"large_image": "asd"
},
"fields": {
"apikey": "hello",
"url": "url"
},
"usage": [{
"workflow_id": "asd",
"nodes": [{
"node_id": ""
}]
}],
"label": "Number 2",
"id": "4f97da9d-1caf-41cc-aa13-67104d8d825d",
"active": true
}
]
}`
*/
}
func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -3712,7 +4124,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
_ = r
log.Printf("Starting workflow folder iteration")
iterateWorkflowGithubFolders(fs, dir, "", "")
iterateWorkflowGithubFolders(fs, dir, "", "", user.Id)
} else if strings.Contains(tmpBody.URL, "s3") {
//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
@@ -4012,7 +4424,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
}
// Onlyname is used to
func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, userId string) error {
var err error
for _, file := range dir {
@@ -4031,7 +4443,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
}
// Go routine? Hmm, this can be super quick I guess
err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "")
err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "", userId)
if err != nil {
continue
}
@@ -4058,6 +4470,11 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
continue
}
// rewrite owner to user who imports now
if userId != "" {
workflow.Owner = userId
}
ctx := context.Background()
err = setWorkflow(ctx, workflow, workflow.ID)
if err != nil {
@@ -4204,6 +4621,54 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
continue
}
// Fixes (appends) authentication parameters if they're required
if workflowapp.Authentication.Required {
log.Printf("Checking authentication fields and appending for %s!", workflowapp.Name)
// FIXME:
// Might require reflection into the python code to append the fields as well
for index, action := range workflowapp.Actions {
if action.AuthNotRequired {
log.Printf("Skipping auth setup: %s", action.Name)
continue
}
// 1. Check if authentication params exists at all
// 2. Check if they're present in the action
// 3. Add them IF they DONT exist
// 4. Fix python code with reflection (FIXME)
appendParams := []WorkflowAppActionParameter{}
for _, fieldname := range workflowapp.Authentication.Parameters {
found := false
for index, param := range action.Parameters {
if param.Name == fieldname.Name {
found = true
action.Parameters[index].Configuration = true
//log.Printf("Set config to true for field %s!", param.Name)
break
}
}
if !found {
appendParams = append(appendParams, WorkflowAppActionParameter{
Name: fieldname.Name,
Description: fieldname.Description,
Example: fieldname.Example,
Required: fieldname.Required,
Configuration: true,
Schema: fieldname.Schema,
})
}
}
if len(appendParams) > 0 {
log.Printf("Appending %d params to the START of %s", len(appendParams), action.Name)
workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...)
}
}
}
err = checkWorkflowApp(workflowapp)
if err != nil {
log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion)
@@ -4252,7 +4717,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
if len(tags) > 0 {
log.Printf("Successfully built image %s", tags[0])
} else {
log.Printf("Successfully built image docker img")
log.Printf("Successfully built Docker image")
}
}
}
@@ -4404,7 +4869,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
}
// Query for the specifci workflowId
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(50)
q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(20)
var workflowExecutions []WorkflowExecution
_, err = dbclient.GetAll(ctx, q, &workflowExecutions)
if err != nil {
@@ -4455,6 +4920,30 @@ func getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) {
return allworkflowapps, nil
}
func getAllWorkflowAppAuth(ctx context.Context) ([]AppAuthenticationStorage, error) {
var allworkflowapps []AppAuthenticationStorage
q := datastore.NewQuery("workflowappauth")
_, err := dbclient.GetAll(ctx, q, &allworkflowapps)
if err != nil {
return []AppAuthenticationStorage{}, err
}
return allworkflowapps, nil
}
func setWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error {
key := datastore.NameKey("workflowappauth", id, nil)
// New struct, to not add body, author etc
if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil {
log.Printf("Error adding workflow app: %s", err)
return err
}
return nil
}
// Hmm, so I guess this should use uuid :(
// Consistency PLX
func setWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error {
docker-compose.yml
+2
View File
@@ -10,6 +10,8 @@ services:
- "${FRONTEND_PORT_HTTPS}:443"
networks:
- shuffle
environment:
- BACKEND_HOSTNAME=${BACKEND_HOSTNAME}
restart: unless-stopped
depends_on:
- backend
frontend/Dockerfile
+18 -4
View File
@@ -8,12 +8,17 @@ COPY package.json /usr/src/app/package.json
RUN npm install --verbose
COPY . /usr/src/app
# copy only required files to not trigger rebuilding every time
COPY ./certs /usr/src/app/certs/
COPY ./public /usr/src/app/public/
COPY ./src /usr/src/app/src/
COPY ./*.sh /usr/src/app/
COPY ./*.json /usr/src/app/
RUN npm run-script build
# Production environment
from nginx:latest
FROM nginx:latest
RUN mkdir -p /usr/share/nginx/html/build
RUN mkdir -p /usr/share/nginx/html/css
@@ -26,8 +31,17 @@ COPY --from=builder /usr/src/app/build /usr/share/nginx/html
COPY --from=builder /usr/src/app/certs/fullchain.pem /etc/nginx/fullchain.cert.pem
COPY --from=builder /usr/src/app/certs/privkey.pem /etc/nginx/privkey.pem
# Prod
COPY --from=builder /usr/src/app/nginx.conf /etc/nginx/nginx.conf
# install CONFD
ENV CONFD_VERSION 0.16.0
RUN apt-get update && apt-get install -y curl && apt-get clean
RUN curl -sSL https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \
chmod +x /usr/local/bin/confd
COPY ./confd /etc/confd
# rewrite command & entrypoint with ours
COPY ./entrypoint.sh /
ENTRYPOINT [ "/entrypoint.sh" ]
CMD ["nginx", "-g", "daemon off;"]
EXPOSE 80
EXPOSE 443
frontend/confd/conf.d/nginx.conf.toml
+9
View File
@@ -0,0 +1,9 @@
[template]
src = "nginx.conf"
dest = "/etc/nginx/nginx.conf"
uid = 0
gid = 0
mode = "0644"
keys = [
"/",
]
frontend/confd/templates/nginx.conf
+103
View File
@@ -0,0 +1,103 @@
user nobody nogroup;
worker_processes auto; # auto-detect number of logical CPU cores
events {
worker_connections 512; # set the max number of simultaneous connections (per worker process)
}
http {
client_max_body_size 250M;
include mime.types;
# thanks stackoverflow http://stackoverflow.com/a/5132440/2406040
gzip on;
gzip_http_version 1.1;
gzip_vary on;
gzip_comp_level 6;
gzip_proxied any;
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript text/xml application/xml application/rss+xml application/atom+xml application/rdf+xml;
# make sure gzip does not lose large gzipped js or css files
# see http://blog.leetsoft.com/2007/07/25/nginx-gzip-ssl.html
gzip_buffers 16 8k;
# Disable gzip for certain browsers.
gzip_disable "MSIE [1-6].(?!.*SV1)";
server {
listen 80;
server_name "localhost";
location / {
# avoid clickjacking
add_header X-Frame-Options DENY;
# block MIME sniffing
add_header X-Content-Type-Options nosniff;
# security headers
add_header X-XSS-Protection "1; mode=block";
# add_header Content-Security-Policy "default-src 'self'";
add_header Referrer-Policy "no-referrer";
server_tokens off;
root /usr/share/nginx/html;
gzip_static on;
expires 1y;
add_header Cache-Control public;
add_header ETag "";
try_files $uri /index.html;
}
location /api/v1 {
proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001;
proxy_buffering off;
proxy_http_version 1.1;
proxy_connect_timeout 900;
proxy_send_timeout 900;
proxy_read_timeout 900;
send_timeout 900;
}
}
server {
listen 443 ssl;
server_name "localhost";
ssl_certificate fullchain.cert.pem;
ssl_certificate_key privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
location / {
# avoid clickjacking
add_header X-Frame-Options DENY;
# block MIME sniffing
add_header X-Content-Type-Options nosniff;
# security headers
add_header X-XSS-Protection "1; mode=block";
# add_header Content-Security-Policy "default-src 'self'";
add_header Referrer-Policy "no-referrer";
server_tokens off;
root /usr/share/nginx/html;
gzip_static on;
expires 1y;
add_header Cache-Control public;
add_header ETag "";
try_files $uri /index.html;
}
# Get the hostname from environment here?
location /api/v1 {
proxy_pass http://{{ getenv "BACKEND_HOSTNAME" "shuffle-backend" }}:5001;
proxy_buffering off;
proxy_http_version 1.1;
proxy_connect_timeout 900;
proxy_send_timeout 900;
proxy_read_timeout 900;
send_timeout 900;
}
}
}
frontend/entrypoint.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/bin/bash
# generate configs
/usr/local/bin/confd -backend="env" -confdir="/etc/confd" -onetime
# run main command
exec "$@"
frontend/nginx.conf
-89
View File
@@ -1,89 +0,0 @@
user nobody nogroup;
worker_processes auto; # auto-detect number of logical CPU cores
events {
worker_connections 512; # set the max number of simultaneous connections (per worker process)
}
http {
client_max_body_size 250M;
include mime.types;
# thanks stackoverflow http://stackoverflow.com/a/5132440/2406040
gzip on;
gzip_http_version 1.1;
gzip_vary on;
gzip_comp_level 6;
gzip_proxied any;
gzip_types text/plain text/css application/json application/javascript application/x-javascript text/javascript text/xml application/xml application/rss+xml application/atom+xml application/rdf+xml;
# make sure gzip does not lose large gzipped js or css files
# see http://blog.leetsoft.com/2007/07/25/nginx-gzip-ssl.html
gzip_buffers 16 8k;
# Disable gzip for certain browsers.
gzip_disable “MSIE [1-6].(?!.*SV1)”;
server {
listen 80;
server_name "localhost";
location / {
# avoid clickjacking
add_header X-Frame-Options DENY;
# block MIME sniffing
add_header X-Content-Type-Options nosniff;
# security headers
add_header X-XSS-Protection "1; mode=block";
# add_header Content-Security-Policy "default-src 'self'";
add_header Referrer-Policy "no-referrer";
server_tokens off;
root /usr/share/nginx/html;
gzip_static on;
expires 1y;
add_header Cache-Control public;
add_header ETag "";
try_files $uri /index.html;
}
location /api/v1 {
proxy_pass http://shuffle-backend:5001;
}
}
server {
listen 443 ssl;
server_name "localhost";
ssl_certificate fullchain.cert.pem;
ssl_certificate_key privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
location / {
# avoid clickjacking
add_header X-Frame-Options DENY;
# block MIME sniffing
add_header X-Content-Type-Options nosniff;
# security headers
add_header X-XSS-Protection "1; mode=block";
# add_header Content-Security-Policy "default-src 'self'";
add_header Referrer-Policy "no-referrer";
server_tokens off;
root /usr/share/nginx/html;
gzip_static on;
expires 1y;
add_header Cache-Control public;
add_header ETag "";
try_files $uri /index.html;
}
# Get the hostname from environment here?
location /api/v1 {
proxy_pass http://shuffle-backend:5001;
}
}
}
frontend/src/Admin.js
+282 -22
View File
@@ -21,6 +21,7 @@ import DialogTitle from '@material-ui/core/DialogTitle';
import DialogActions from '@material-ui/core/DialogActions';
import DialogContent from '@material-ui/core/DialogContent';
import CachedIcon from '@material-ui/icons/Cached';
const surfaceColor = "#27292D"
const inputColor = "#383B40"
@@ -33,13 +34,45 @@ const Admin = (props) => {
const [curTab, setCurTab] = React.useState(0);
const [users, setUsers] = React.useState([]);
const [environments, setEnvironments] = React.useState([]);
const [authentication, setAuthentication] = React.useState([]);
const [schedules, setSchedules] = React.useState([])
const [selectedUser, setSelectedUser] = React.useState({})
const [newPassword, setNewPassword] = React.useState("");
const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false)
const [selectedAuthentication, setSelectedAuthentication] = React.useState({})
const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false)
const alert = useAlert()
const deleteAuthentication = (data) => {
alert.info("Deleting auth "+data.label)
// Just use this one?
const url = globalUrl+'/api/v1/apps/authentication/'+data.id
console.log("URL: ", url)
fetch(url, {
method: 'DELETE',
credentials: "include",
headers: {
'Content-Type': 'application/json',
},
})
.then(response =>
response.json().then(responseJson => {
console.log("RESP: ", responseJson)
if (responseJson["success"] === false) {
alert.error("Failed stopping schedule")
} else {
getAppAuthentication()
alert.success("Successfully stopped schedule!")
}
}),
)
.catch(error => {
console.log("Error in userdata: ", error)
});
}
const deleteSchedule = (data) => {
// FIXME - add some check here ROFL
console.log("INPUT: ", data)
@@ -162,6 +195,7 @@ const Admin = (props) => {
const deleteEnvironment = (name) => {
// FIXME - add some check here ROFL
alert.info("Deleting environment "+name)
var newEnv = []
for (var key in environments) {
if (environments[key].Name == name) {
@@ -253,6 +287,36 @@ const Admin = (props) => {
});
}
const getAppAuthentication = () => {
fetch(globalUrl+"/api/v1/apps/authentication", {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
return
}
return response.json()
})
.then((responseJson) => {
if (responseJson.success) {
console.log(responseJson.data)
setAuthentication(responseJson.data)
} else {
alert.error("Failed getting authentications")
}
})
.catch(error => {
alert.error(error.toString())
});
}
const getEnvironments = () => {
fetch(globalUrl+"/api/v1/getenvironments", {
method: 'GET',
@@ -271,6 +335,7 @@ const Admin = (props) => {
return response.json()
})
.then((responseJson) => {
console.log(responseJson)
setEnvironments(responseJson)
})
.catch(error => {
@@ -393,6 +458,70 @@ const Admin = (props) => {
});
}
const editAuthenticationModal =
<Dialog modal
open={selectedAuthenticationModalOpen}
onClose={() => {setSelectedAuthenticationModalOpen(false)}}
PaperProps={{
style: {
backgroundColor: surfaceColor,
color: "white",
minWidth: "800px",
minHeight: "320px",
},
}}
>
<DialogTitle><span style={{color: "white"}}>Edit authentication</span></DialogTitle>
<DialogContent>
<div style={{display: "flex"}}>
<TextField
style={{backgroundColor: inputColor, flex: 3}}
InputProps={{
style:{
height: 50,
color: "white",
},
}}
color="primary"
required
fullWidth={true}
placeholder="New password"
type="password"
id="standard-required"
autoComplete="password"
margin="normal"
variant="outlined"
onChange={e => setNewPassword(e.target.value)}
/>
<Button
style={{maxHeight: 50, flex: 1}}
variant="outlined"
color="primary"
onClick={() => onPasswordChange()}
>
Submit
</Button>
</div>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: inputColor}}/>
<Button
style={{}}
variant="outlined"
color="primary"
onClick={() => deleteUser(selectedUser)}
>
{selectedUser.active ? "Deactivate" : "Activate"}
</Button>
<Button
style={{}}
variant="outlined"
color="primary"
onClick={() => generateApikey(selectedUser.id)}
>
Get new API key
</Button>
</DialogContent>
</Dialog>
const editUserModal =
<Dialog modal
open={selectedUserModalOpen}
@@ -410,7 +539,7 @@ const Admin = (props) => {
<DialogContent>
<div style={{display: "flex"}}>
<TextField
style={{backgroundColor: inputColor, flex: 3}}
style={{marginTop: 0, backgroundColor: inputColor, flex: 3}}
InputProps={{
style:{
height: 50,
@@ -470,7 +599,9 @@ const Admin = (props) => {
},
}}
>
<DialogTitle><span style={{color: "white"}}>Add user</span></DialogTitle>
<DialogTitle><span style={{color: "white"}}>
{curTab === 0 ? "Add user" : "Add environment"}
</span></DialogTitle>
<DialogContent>
{curTab === 0 ?
<div>
@@ -517,7 +648,7 @@ const Admin = (props) => {
onChange={(event) => changeModalData("Password", event.target.value)}
/>
</div>
: curTab === 1 ?
: curTab === 2 ?
<div>
Environment Name
<TextField
@@ -550,7 +681,7 @@ const Admin = (props) => {
<Button variant="contained" style={{borderRadius: "0px"}} onClick={() => {
if (curTab === 0) {
submitUser(modalUser)
} else if (curTab === 1) {
} else if (curTab === 2) {
submitEnvironment(modalUser)
}
}} color="primary">
@@ -561,9 +692,11 @@ const Admin = (props) => {
const usersView = curTab === 0 ?
<div>
<h2>
User management
</h2>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>User management</h2>
<span style={{marginLeft: 25}}>Add, edit, block or change passwords</span>
</div>
<div/>
<Button
style={{}}
variant="contained"
@@ -660,11 +793,12 @@ const Admin = (props) => {
</div>
: null
const schedulesView = curTab === 2 ?
const schedulesView = curTab === 3 ?
<div>
<h2>
Schedules
</h2>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Schedules</h2>
<span style={{marginLeft: 25}}>Schedules used in Workflows. Makes locating and control easier.</span>
</div>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: inputColor}}/>
<List>
<ListItem>
@@ -708,11 +842,97 @@ const Admin = (props) => {
</div>
: null
const environmentView = curTab === 1 ?
const authenticationView = curTab === 1 ?
<div>
<h2>
Environments
</h2>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>App Authentication</h2>
<span style={{marginLeft: 25}}>Control the authentication options for individual apps. <b>Actions can be destructive!</b></span>
</div>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: inputColor}}/>
<List>
<ListItem>
<ListItemText
primary="Icon"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Label"
style={{minWidth: 250, maxWidth: 250}}
/>
<ListItemText
primary="App Name"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Workflows"
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
<ListItemText
primary="Action amount"
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
<ListItemText
primary="Fields"
style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}}
/>
<ListItemText
primary="Actions"
/>
</ListItem>
{authentication === undefined ? null : authentication.map(data => {
return (
<ListItem>
<ListItemText
primary=<img alt="" src={data.app.large_image} style={{maxWidth: 50,}} />
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.label}
style={{minWidth: 250, maxWidth: 250}}
/>
<ListItemText
primary={data.app.name}
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.usage.length}
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
<ListItemText
primary={data.node_count}
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
<ListItemText
primary={data.fields.map(data => {
return data.key
}).join(", ")}
style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}}
/>
<ListItemText>
<Button
style={{}}
variant="outlined"
color="primary"
onClick={() => {
deleteAuthentication(data)
}}
>
Delete
</Button>
</ListItemText>
</ListItem>
)
})}
</List>
</div>
: null
const environmentView = curTab === 2 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Environments</h2>
<span style={{marginLeft: 25}}>Decides what Orborus environment to execute an action in a workflow in.</span>
</div>
<Button
style={{}}
variant="contained"
@@ -721,13 +941,44 @@ const Admin = (props) => {
>
Add environment
</Button>
<Button
style={{marginLeft: 5, }}
variant="contained"
color="primary"
onClick={() => getEnvironments()}
>
<CachedIcon />
</Button>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: inputColor}}/>
<List>
<ListItem>
<ListItemText
primary="Name"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Orborus running (TBD)"
style={{minWidth: 200, maxWidth: 200}}
/>
<ListItemText
primary="Actions"
style={{minWidth: 150, maxWidth: 150}}
/>
</ListItem>
{environments === undefined ? null : environments.map(environment => {
return (
<ListItem>
<Button type="outlined" style={{borderRadius: "0px"}} onClick={() => deleteEnvironment(environment.Name)} color="primary">Delete</Button>
- {environment.Name}
<ListItemText
primary={environment.Name}
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
<ListItemText
primary={"TBD"}
style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}}
/>
<ListItemText>
<Button type="outlined" style={{borderRadius: "0px"}} onClick={() => deleteEnvironment(environment.Name)} color="primary">Delete</Button>
</ListItemText>
</ListItem>
)
})}
@@ -735,10 +986,14 @@ const Admin = (props) => {
</div>
: null
// primary={environment.Registered ? "true" : "false"}
const setConfig = (event, newValue) => {
if (newValue === 1) {
getEnvironments()
getAppAuthentication()
} else if (newValue === 2) {
getEnvironments()
} else if (newValue === 3) {
getSchedules()
}
@@ -757,13 +1012,17 @@ const Admin = (props) => {
aria-label="disabled tabs example"
>
<Tab label="Users" />
<Tab label="App Authentication"/>
<Tab label="Environments"/>
<Tab label="Schedules"/>
</Tabs>
<div style={{marginBottom: 10}}/>
{usersView}
{environmentView}
{schedulesView}
<Divider style={{marginTop: 0, marginBottom: 10, backgroundColor: "rgb(91, 96, 100)"}} />
<div style={{padding: 15}}>
{authenticationView}
{usersView}
{environmentView}
{schedulesView}
</div>
</Paper>
</div>
@@ -771,6 +1030,7 @@ const Admin = (props) => {
<div>
{modalView}
{editUserModal}
{editAuthenticationModal}
{data}
</div>
)
frontend/src/AngularWorkflow.js
+614 -137
View File
File diff suppressed because it is too large Load Diff
frontend/src/AppCreator.js
+6
View File
@@ -716,6 +716,11 @@ const AppCreator = (props) => {
}
if (authenticationOption === "API key") {
if (parameterName.length === 0) {
alert.error("A field name for the APIkey must be defined")
return
}
data.components.securitySchemes["ApiKeyAuth"] = {
"type": "apiKey",
"in": parameterLocation.toLowerCase(),
@@ -727,6 +732,7 @@ const AppCreator = (props) => {
"scheme": "bearer",
"bearerFormat": "UUID",
}
} else if (authenticationOption === "Basic auth") {
data.components.securitySchemes["BasicAuth"] = {
"type": "http",
frontend/src/Apps.js
+103 -1
View File
@@ -19,6 +19,7 @@ import Input from '@material-ui/core/Input';
import YAML from 'yaml'
import {Link} from 'react-router-dom';
import Breadcrumbs from '@material-ui/core/Breadcrumbs';
import ReactJson from 'react-json-view'
import CachedIcon from '@material-ui/icons/Cached';
import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
@@ -39,6 +40,51 @@ import CircularProgress from '@material-ui/core/CircularProgress';
const surfaceColor = "#27292D"
const inputColor = "#383B40"
// Parses JSON data into keys that can be used everywhere :)
export const GetParsedPaths = (inputdata, basekey) => {
const splitkey = " => "
var parsedValues = []
for (const [key, value] of Object.entries(inputdata)) {
// Check if loop or JSON
const extra = basekey.length > 0 ? splitkey : ""
const basekeyname = `${basekey.slice(1,basekey.length).split(".").join(splitkey)}${extra}${key}`
if (typeof(value) === 'object') {
if (Array.isArray(value)) {
// Check if each item is object
parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`})
parsedValues.push({"type": "list", "name": `${basekeyname}${splitkey}list`, "autocomplete": `${basekey}.${key}.#`})
// Only check the first. This would be probably be dumb otherwise.
for (var subkey in value) {
if (typeof(value) === 'object') {
const returnValues = GetParsedPaths(value[subkey], `${basekey}.${key}.#`)
for (var subkey in returnValues) {
parsedValues.push(returnValues[subkey])
}
}
// Don't need else as # (all items) is already defined before the loop
break
}
//console.log(key+" is array")
} else {
parsedValues.push({"type": "object", "name": basekeyname, "autocomplete": `${basekey}.${key}`})
const returnValues = GetParsedPaths(value, `${basekey}.${key}`)
for (var subkey in returnValues) {
parsedValues.push(returnValues[subkey])
}
}
} else {
parsedValues.push({"type": "value", "name": basekeyname, "autocomplete": `${basekey}.${key}`, "value": value,})
}
}
return parsedValues
}
const Apps = (props) => {
const { globalUrl, isLoggedIn, isLoaded } = props;
@@ -243,8 +289,8 @@ const Apps = (props) => {
<Paper square style={paperAppStyle} onClick={() => {
if (selectedApp.id !== data.id) {
setSelectedApp(data)
console.log(data)
if (data.actions !== undefined && data.actions !== null && data.actions.length > 0) {
console.log(data.actions[0])
setSelectedAction(data.actions[0])
} else {
setSelectedAction({})
@@ -367,6 +413,61 @@ const Apps = (props) => {
:
<img alt={selectedApp.title} src={selectedApp.large_image} style={{width: 100, height: 100, maxWidth: "100%"}} />
const GetAppExample = () => {
if (selectedAction.returns === undefined) {
return null
}
var showResult = selectedAction.returns.example
if (showResult === undefined || showResult === null || showResult.length === 0) {
return null
}
var jsonvalid = true
try {
const tmp = String(JSON.parse(showResult))
if (!tmp.includes("{") && !tmp.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
// FIXME: In here -> parse the values into a list or something
if (jsonvalid) {
const paths = GetParsedPaths(JSON.parse(showResult), "")
console.log("PATHS: ", paths)
return (
<div>
{paths.map(data => {
const circleSize = 10
return (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data} onClick={() => console.log(data.autocomplete)}>
{data.name}
</MenuItem>
)
})}
<ReactJson
src={JSON.parse(showResult)}
theme="solarized"
collapsed={false}
displayDataTypes={true}
name={"Example return value"}
/>
</div>
)
}
return (
<div>
<b>Example return</b><div/>
{selectedAction.returns.example}
</div>
)
}
//fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), {
var baseInfo = newAppname.length > 0 ?
<div>
@@ -454,6 +555,7 @@ const Apps = (props) => {
{selectedAction.description}
</div>
: null}
<GetAppExample />
</div>
:
null
frontend/src/Workflows.js
+11 -6
View File
@@ -180,15 +180,20 @@ const Workflows = (props) => {
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for WORKFLOW EXECUTION :O!")
alert.error("Failed loading executions for current workflow")
}
return response.json()
})
.then((responseJson) => {
setWorkflowExecutions(responseJson)
if (responseJson.length > 0) {
setSelectedExecution(responseJson[0])
if (responseJson.success === false) {
alert.error("Failed getting executions")
} else {
if (responseJson.length > 0) {
setSelectedExecution(responseJson[0])
setWorkflowExecutions(responseJson)
} else {
alert.info("Couldn't find executions for the workflow")
}
}
})
.catch(error => {
@@ -949,9 +954,9 @@ const Workflows = (props) => {
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
{workflows.map(data => {
{workflows.map((data, index) => {
return (
<WorkflowPaper data={data} />
<WorkflowPaper key={index} data={data} />
)
})}
</div>
frontend/src/defaultCytoscapeStyle.js
+11 -1
View File
@@ -172,10 +172,20 @@ const data = [{
'background-color': '#f4ad42',
'border-color': '#f4ad42',
'border-width': '5px',
'transition-property': 'background-color',
'transition-property': 'border-color',
'transition-duration': '0.5s',
},
},
{
selector: '.shuffle-hover-highlight',
css: {
'background-color': "#f85a3e",
'border-color': '#f85a3e',
'border-width': '5px',
'transition-property': 'border-width',
'transition-duration': '0.25s',
},
},
{
selector: '$node > node',
css: {
functions/onprem/orborus/orborus.go
+52 -6
View File
@@ -27,6 +27,7 @@ import (
var baseUrl = os.Getenv("BASE_URL")
var baseimagename = "frikky/shuffle"
var shuffleNetwork = "" // Filled in init if found
var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
var environment = os.Getenv("ENVIRONMENT_NAME")
@@ -61,6 +62,49 @@ func init() {
if err != nil {
panic(fmt.Sprintf("Unable to create docker client: %s", err))
}
// FIXME: Move this to global variables?
containerIdentifier := "orborus"
networkIdentifier := "shuffle"
ctx := context.Background()
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
if err != nil {
log.Printf("Failed getting containers during init - running without network check: %s", err)
}
// Skip random containers. Only handle things related to Shuffle.
for _, container := range containers {
found := false
//log.Printf("Running? %#v", container)
if container.State != "running" {
continue
}
for _, name := range container.Names {
if !strings.Contains(strings.ToLower(name), containerIdentifier) {
found = true
continue
}
}
if found {
for key, _ := range container.NetworkSettings.Networks {
if strings.Contains(strings.ToLower(key), networkIdentifier) {
shuffleNetwork = key
break
}
}
}
}
if len(shuffleNetwork) > 0 {
log.Printf("Found shuffle network \"%s\" for container %s", shuffleNetwork, containerIdentifier)
} else {
log.Printf("Running Shuffle without a docker network")
}
}
// Deploys the internal worker whenever something happens
@@ -82,18 +126,20 @@ func deployWorker(image string, identifier string, env []string) {
Env: env,
}
// Set the network
// Look for Shuffle network and set it
// FIXME: Move this out of here and have it be a global setting. During init?
networkConfig := &network.NetworkingConfig{}
if baseUrl == "http://shuffle-backend:5001" {
if len(shuffleNetwork) > 0 {
networkConfig = &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"shuffle_shuffle": {
NetworkID: "shuffle_shuffle",
shuffleNetwork: {
NetworkID: shuffleNetwork,
},
},
}
} else {
// USE PROXY
env = append(env, fmt.Sprintf("DOCKER_NETWORK", shuffleNetwork))
}
//test := &network.EndpointSettings{
functions/onprem/worker/worker.go
+215 -128
View File
@@ -25,29 +25,34 @@ var baseUrl = os.Getenv("BASE_URL")
var baseimagename = "frikky/shuffle"
var sleepTime = 2
type Condition struct {
AppName string `json:"app_name"`
AppVersion string `json:"app_version"`
Conditional string `json:"conditional"`
Errors []string `json:"errors"`
ID string `json:"id"`
IsValid bool `json:"is_valid"`
Label string `json:"label"`
Name string `json:"name"`
Position struct {
X float64 `json:"x"`
Y float64 `json:"y"`
} `json:"position"`
type User struct {
Username string `datastore:"Username" json:"username"`
Password string `datastore:"password,noindex" password:"password,omitempty"`
Session string `datastore:"session,noindex" json:"session"`
Verified bool `datastore:"verified,noindex" json:"verified"`
PrivateApps []WorkflowApp `datastore:"privateapps" json:"privateapps":`
Role string `datastore:"role" json:"role"`
Roles []string `datastore:"roles" json:"roles"`
VerificationToken string `datastore:"verification_token" json:"verification_token"`
ApiKey string `datastore:"apikey" json:"apikey"`
ResetReference string `datastore:"reset_reference" json:"reset_reference"`
ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"`
Id string `datastore:"id" json:"id"`
Orgs []string `datastore:"orgs" json:"orgs"`
CreationTime int64 `datastore:"creation_time" json:"creation_time"`
Active bool `datastore:"active" json:"active"`
}
type User struct {
Username string `datastore:"Username"`
Password string `datastore:"password,noindex"`
Session string `datastore:"session,noindex"`
Verified bool `datastore:"verified,noindex"`
ApiKey string `datastore:"apikey,noindex"`
Id string `datastore:"id" json:"id"`
Orgs string `datastore:"orgs" json:"orgs"`
type ExecutionRequest struct {
ExecutionId string `json:"execution_id"`
ExecutionArgument string `json:"execution_argument"`
ExecutionSource string `json:"execution_source"`
WorkflowId string `json:"workflow_id"`
Environments []string `json:"environments"`
Authorization string `json:"authorization"`
Status string `json:"status"`
Start string `json:"start"`
Type string `json:"type"`
}
type Org struct {
@@ -57,12 +62,106 @@ type Org struct {
Id string `json:"id"`
}
type AppAuthenticationStorage struct {
Active bool `json:"active" datastore:"active"`
Label string `json:"label" datastore:"label"`
Id string `json:"id" datastore:"id"`
App WorkflowApp `json:"app" datastore:"app"`
Fields []AuthenticationStore `json:"fields" datastore:"fields"`
Usage []AuthenticationUsage `json:"usage" datastore:"usage"`
WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"`
NodeCount int64 `json:"node_count" datastore:"node_count"`
}
type AuthenticationUsage struct {
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
Nodes []string `json:"nodes" datastore:"nodes"`
}
// An app inside Shuffle
type WorkflowApp struct {
Name string `json:"name" yaml:"name" required:true datastore:"name"`
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"`
Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"`
Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"`
Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"`
Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"`
Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"`
Owner string `json:"owner" datastore:"owner" yaml:"owner"`
Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps
PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"`
Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"`
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
ContactInfo struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Url string `json:"url" datastore:"url" yaml:"url"`
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
}
type WorkflowAppActionParameter struct {
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
Required bool `json:"required" datastore:"required" yaml:"required"`
Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
}
type SchemaDefinition struct {
Type string `json:"type" datastore:"type"`
}
type WorkflowAppAction struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name"`
Label string `json:"label" datastore:"label"`
NodeType string `json:"node_type" datastore:"node_type"`
Environment string `json:"environment" datastore:"environment"`
Sharing bool `json:"sharing" datastore:"sharing"`
PrivateID string `json:"private_id" datastore:"private_id"`
AppID string `json:"app_id" datastore:"app_id"`
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
Tested bool `json:"tested" datastore:"tested" yaml:"tested"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
ExecutionVariable struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variable" datastore:"execution_variables"`
Returns struct {
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
Example string `json:"example" datastore:"example" yaml:"example"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"returns" datastore:"returns"`
AuthenticationId string `json:"authentication_id" datastore:"authentication_id"`
Example string `json:"example" datastore:"example" yaml:"example"`
AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"`
}
// FIXME: Generate a callback authentication ID?
type WorkflowExecution struct {
Type string `json:"type" datastore:"type"`
Status string `json:"status" datastore:"status"`
Start string `json:"start" datastore:"start"`
ExecutionArgument string `json:"execution_argument" datastore:"execution_argument"`
ExecutionId string `json:"execution_id" datastore:"execution_id"`
ExecutionSource string `json:"execution_source" datastore:"execution_source"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id"`
LastNode string `json:"last_node" datastore:"last_node"`
Authorization string `json:"authorization" datastore:"authorization"`
@@ -81,6 +180,7 @@ type WorkflowExecution struct {
} `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"`
}
// This is for the nodes in a workflow, NOT the app action itself.
type Action struct {
AppName string `json:"app_name" datastore:"app_name"`
AppVersion string `json:"app_version" datastore:"app_version"`
@@ -107,14 +207,51 @@ type Action struct {
X float64 `json:"x" datastore:"x"`
Y float64 `json:"y" datastore:"y"`
} `json:"position"`
Priority int `json:"priority" datastore:"priority"`
AuthenticationId string `json:"authentication_id" datastore:"authentication_id"`
Example string `json:"example" datastore:"example"`
AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"`
}
// Added environment for location to execute
type Trigger struct {
AppName string `json:"app_name" datastore:"app_name"`
Description string `json:"description" datastore:"description"`
LongDescription string `json:"long_description" datastore:"long_description"`
Status string `json:"status" datastore:"status"`
AppVersion string `json:"app_version" datastore:"app_version"`
Errors []string `json:"errors" datastore:"errors"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
IsStartNode bool `json:"isStartNode" datastore:"isStartNode"`
Label string `json:"label" datastore:"label"`
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
Environment string `json:"environment" datastore:"environment"`
TriggerType string `json:"trigger_type" datastore:"trigger_type"`
Name string `json:"name" datastore:"name"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"`
Position struct {
X float64 `json:"x" datastore:"x"`
Y float64 `json:"y" datastore:"y"`
} `json:"position"`
Priority int `json:"priority" datastore:"priority"`
}
type Branch struct {
DestinationID string `json:"destination_id" datastore:"destination_id"`
ID string `json:"id" datastore:"id"`
SourceID string `json:"source_id" datastore:"source_id"`
HasError bool `json:"has_errors" datastore: "has_errors"`
DestinationID string `json:"destination_id" datastore:"destination_id"`
ID string `json:"id" datastore:"id"`
SourceID string `json:"source_id" datastore:"source_id"`
Label string `json:"label" datastore:"label"`
HasError bool `json:"has_errors" datastore: "has_errors"`
Conditions []Condition `json:"conditions" datastore: "conditions"`
}
// Same format for a lot of stuff
type Condition struct {
Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"`
Source WorkflowAppActionParameter `json:"source" datastore:"source"`
Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"`
}
type Schedule struct {
@@ -124,44 +261,26 @@ type Schedule struct {
Id string `json:"id" datastore:"id"`
}
type Trigger struct {
AppName string `json:"app_name" datastore:"app_name"`
Status string `json:"status" datastore:"status"`
AppVersion string `json:"app_version" datastore:"app_version"`
Errors []string `json:"errors" datastore:"errors"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
IsStartNode bool `json:"isStartNode" datastore:"isStartNode"`
Label string `json:"label" datastore:"label"`
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
Environment string `json:"environment" datastore:"environment"`
TriggerType string `json:"trigger_type" datastore:"trigger_type"`
Name string `json:"name" datastore:"name"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
Position struct {
X float64 `json:"x" datastore:"x"`
Y float64 `json:"y" datastore:"y"`
} `json:"position"`
Priority int `json:"priority" datastore:"priority"`
}
type Workflow struct {
Actions []Action `json:"actions" datastore:"actions,noindex"`
Branches []Branch `json:"branches" datastore:"branches,noindex"`
Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"`
Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"`
Errors []string `json:"errors,omitempty" datastore:"errors"`
Tags []string `json:"tags,omitempty" datastore:"tags"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
Name string `json:"name" datastore:"name"`
Description string `json:"description" datastore:"description"`
Start string `json:"start" datastore:"start"`
Owner string `json:"owner" datastore:"owner"`
Sharing string `json:"sharing" datastore:"sharing"`
Org []Org `json:"org,omitempty" datastore:"org"`
ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"`
Actions []Action `json:"actions" datastore:"actions,noindex"`
Branches []Branch `json:"branches" datastore:"branches,noindex"`
Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"`
Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"`
Configuration struct {
ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"`
StartFromTop bool `json:"start_from_top" datastore:"start_from_top"`
} `json:"configuration,omitempty" datastore:"configuration"`
Errors []string `json:"errors,omitempty" datastore:"errors"`
Tags []string `json:"tags,omitempty" datastore:"tags"`
ID string `json:"id" datastore:"id"`
IsValid bool `json:"is_valid" datastore:"is_valid"`
Name string `json:"name" datastore:"name"`
Description string `json:"description" datastore:"description"`
Start string `json:"start" datastore:"start"`
Owner string `json:"owner" datastore:"owner"`
Sharing string `json:"sharing" datastore:"sharing"`
Org []Org `json:"org,omitempty" datastore:"org"`
ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"`
WorkflowVariables []struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id"`
@@ -173,48 +292,35 @@ type Workflow struct {
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"`
} `json:"execution_variables,omitempty" datastore:"execution_variables"`
}
type ActionResult struct {
Action Action `json:"action" datastore:"action"`
ExecutionId string `json:"execution_id" datastore:"execution_id"`
Authorization string `json:"authorization" datastore:"authorization"`
Result string `json:"result" datastore:"result"`
Result string `json:"result" datastore:"result,noindex"`
StartedAt int64 `json:"started_at" datastore:"started_at"`
CompletedAt int64 `json:"completed_at" datastore:"completed_at"`
Status string `json:"status" datastore:"status"`
}
type WorkflowApp struct {
Name string `json:"name" yaml:"name" required:true datastore:"name"`
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
ID string `json:"id" yaml:"id" required:false datastore:"id"`
Link string `json:"link" yaml:"link" required:false datastore:"link"`
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
Description string `json:"description" datastore:"description" required:false yaml:"description"`
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
ContactInfo struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Url string `json:"url" datastore:"url" yaml:"url"`
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
type Authentication struct {
Required bool `json:"required" datastore:"required" yaml:"required" `
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
}
// Name = current field
// action_field is the field that it's set to
// value, if Variant = ACTION_RESULT = the second field thingy, which will be
type WorkflowAppActionParameter struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
ActionField string `json:"action_field" datastore:"action_field"`
Variant string `json:"variant", datastore:"variant"`
Required bool `json:"required" datastore:"required"`
Schema struct {
Type string `json:"type" datastore:"type"`
} `json:"schema"`
type AuthenticationParams struct {
Description string `json:"description" datastore:"description" yaml:"description"`
ID string `json:"id" datastore:"id" yaml:"id"`
Name string `json:"name" datastore:"name" yaml:"name"`
Example string `json:"example" datastore:"example" yaml:"example"`
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
Required bool `json:"required" datastore:"required" yaml:"required"`
In string `json:"in" datastore:"in" yaml:"in"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated
}
type AuthenticationStore struct {
@@ -222,34 +328,8 @@ type AuthenticationStore struct {
Value string `json:"value" datastore:"value"`
}
type WorkflowAppAction struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Name string `json:"name" datastore:"name"`
Label string `json:"label" datastore:"label"`
NodeType string `json:"node_type" datastore:"node_type"`
Environment string `json:"environment" datastore:"environment"`
Sharing bool `json:"sharing" datastore:"sharing"`
PrivateID string `json:"private_id" datastore:"private_id"`
AppID string `json:"app_id" datastore:"app_id"`
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
Tested bool `json:"tested" datastore:"tested" yaml:"tested"`
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
ExecutionVariable struct {
Description string `json:"description" datastore:"description"`
ID string `json:"id" datastore:"id"`
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value"`
} `json:"execution_variable" datastore:"execution_variables"`
Returns struct {
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"returns" datastore:"returns"`
}
type SchemaDefinition struct {
Type string `json:"type" datastore:"type"`
type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
}
// removes every container except itself (worker)
@@ -347,17 +427,15 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
}
networkConfig := &network.NetworkingConfig{}
if baseUrl == "http://shuffle-backend:5001" {
shuffleNetwork := os.Getenv("DOCKER_NETWORK")
if len(shuffleNetwork) > 0 {
networkConfig = &network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"shuffle_shuffle": {
NetworkID: "shuffle_shuffle",
shuffleNetwork: {
NetworkID: shuffleNetwork,
},
},
}
} else {
// FIXME: Default config
//log.Printf("Bad config: %s. Using default network", baseUrl)
}
cont, err := cli.ContainerCreate(
@@ -785,14 +863,21 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W
continue
}
//log.Println(string(actionData))
// FIXME - add proper FUNCTION_APIKEY from user definition
executionData, err := json.Marshal(workflowExecution)
if err != nil {
log.Printf("Failed marshalling executiondata: %s", err)
executionData = []byte("")
}
// Sending full execution so that it won't have to load in every app
// This might be an issue if they can read environments, but that's alright
// if everything is generated during execution
env := []string{
fmt.Sprintf("ACTION=%s", string(actionData)),
fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId),
fmt.Sprintf("FUNCTION_APIKEY=%s", "asdasd"),
fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization),
fmt.Sprintf("CALLBACK_URL=%s", baseUrl),
fmt.Sprintf("FULL_EXECUTION=%s", string(executionData)),
}
err = deployApp(dockercli, image, identifier, env)
@@ -1064,6 +1149,8 @@ func main() {
}
for {
// Because of this, it always has updated data.
// Removed request requirement from app_sdk
newresp, err := client.Do(req)
if err != nil {
log.Printf("Failed request: %s", err)