Merge pull request #234 from frikky/launch

Shuffle v0.56 - Scalability and file fixing
This commit is contained in:
Frikky
2021-01-27 15:32:28 +01:00
committed by GitHub
42 changed files with 8835 additions and 2051 deletions
+4 -1
View File
@@ -39,4 +39,7 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_NAME=frikky
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.0"
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.3"
# Used for auto-cleanup of containers. REALLY important at scale.
SHUFFLE_CONTAINER_AUTO_CLEANUP=false
+2 -2
View File
@@ -2,8 +2,8 @@
github: frikky
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
open_collective: shuffle
ko_fi: frikky
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
+13
View File
@@ -43,6 +43,19 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if
## Website
https://shuffler.io
## Contributors
![ICPL logo](https://github.com/frikky/Shuffle/blob/launch/frontend/src/assets/img/icpl_logo.png)
**Shuffle**
<a href="https://github.com/frikky/shuffle/graphs/contributors">
<img src="https://contrib.rocks/image?repo=frikky/shuffle" />
</a>
[**App magicians**](https://github.com/frikky/shuffle-apps)
<a href="https://github.com/frikky/shuffle-apps/graphs/contributors">
<img src="https://contrib.rocks/image?repo=frikky/shuffle-apps" />
</a>
## License
All modular information related to Shuffle will be under MIT (anyone can use it for whatever purpose), with Shuffle itself using AGPLv3.
+1 -1
View File
@@ -7,7 +7,7 @@ RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
RUN pip3 install -r /requirements.txt
FROM base
File diff suppressed because it is too large Load Diff
+2 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.2
VERSION=0.8.54
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
@@ -8,6 +8,7 @@ docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.
#docker push frikky/$NAME:$VERSION
#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
#docker push ghcr.io/frikky/$NAME:$VERSION
#docker tag ghcr.io/frikky/$NAME:$VERSION frikky/shuffle:app_sdk
docker push frikky/shuffle:app_sdk
docker push ghcr.io/frikky/$NAME:$VERSION
+1 -1
View File
@@ -1,2 +1,2 @@
requests
urllib3
requests
+90 -18
View File
@@ -244,7 +244,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
// This function generates the python code that's being used.
// This is really meta when you program it. Handling parameters is hard here.
func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) {
func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) {
method = strings.ToLower(method)
queryString := ""
queryData := ""
@@ -365,21 +365,39 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
preparedHeaders += "}"
}
fileBalance := ""
fileAdder := ``
fileGrabber := ``
fileParameter := ``
if method == "post" && len(fileField) > 0 {
fileParameter = ", file_id"
fileGrabber = `filedata = self.get_file(file_id)`
// This indentation is confusing (but correct) ROFL
fileAdder = fmt.Sprintf(`if not filedata["success"]:
return file_id+" is not a valid File ID"
files = {"%s": (filedata["filename"], filedata["data"])}`, fileField)
fileBalance = ", files=files"
}
// Extra param for url if it's changeable
// Extra param for authentication scheme(s)
// The last weird one is the body.. Tabs & spaces sucks.
data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s):
data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s):
%s
url=f"%s%s"
%s
%s
%s
%s
return requests.%s(url, headers=headers%s%s%s).text
%s
%s
return requests.%s(url, headers=headers%s%s%s%s).text
`,
functionname,
authenticationParameter,
urlParameter,
fileParameter,
parameterData,
queryString,
bodyParameter,
@@ -391,18 +409,20 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
authenticationSetup,
queryData,
bodyFormatter,
fileGrabber,
fileAdder,
method,
authenticationAddin,
bodyAddin,
verifyAddin,
fileBalance,
)
/*
if strings.Contains(functionname, "search") {
log.Println(data)
log.Printf("Queries: %s", queryString)
}
*/
if strings.Contains(functionname, "filescan") {
//log.Printf("FUNCTION: %s", data)
log.Println(data)
log.Printf("Queries: %s", queryString)
}
//log.Printf(data)
return functionname, data
@@ -446,6 +466,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
api.Sharing = false
api.Verified = false
api.Tested = false
api.Invalid = false
api.PrivateID = newmd5
api.Generated = true
api.Activated = true
@@ -638,10 +659,15 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
// This is the python code to be generated
// Could just as well be go at this point lol
pythonFunctions := []string{}
//Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"`
for actualPath, path := range swagger.Paths {
actualPath = strings.Replace(actualPath, " ", "_", -1)
//actualPath = strings.Replace(actualPath, ".", "", -1)
actualPath = strings.Replace(actualPath, "\\", "", -1)
if !api.Invalid && strings.HasPrefix(actualPath, "tmp") {
log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath)
api.Invalid = true
}
// FIXME: Handle everything behind questionmark (?) with dots as well.
// https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem
@@ -846,10 +872,10 @@ def run(request):
func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error {
err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
if err != nil {
log.Printf("Failed setting workflowapp: %s", err)
log.Printf("[ERROR] Failed setting workflowapp: %s", err)
return err
} else {
log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
}
return nil
@@ -948,6 +974,12 @@ func validateParameterName(name string) string {
}
}
newname = strings.ReplaceAll(newname, " ", "_")
newname = strings.ReplaceAll(newname, ",", "_")
newname = strings.ReplaceAll(newname, ".", "_")
newname = strings.ReplaceAll(newname, "|", "_")
newname = strings.ReplaceAll(newname, "-", "_")
return newname
}
@@ -1001,6 +1033,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
parsedName = strings.ReplaceAll(parsedName, "-", "_")
parsedName = validateParameterName(parsedName)
param.Value.Name = parsedName
path.Connect.Parameters[counter].Value.Name = parsedName
@@ -1076,7 +1109,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1211,7 +1244,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1344,7 +1377,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1478,7 +1511,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1521,6 +1554,40 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
},
})
fileField := ""
if path.Post.RequestBody != nil {
//log.Printf("DATA: %#v",
value := path.Post.RequestBody.Value
//log.Printf("VAL: %#v", value.Content)
if val, ok := value.Content["multipart/form-data"]; ok {
if val.Schema.Value != nil {
if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok {
if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok {
fieldname := extensionvalue.(json.RawMessage)
newName := string(fmt.Sprintf("%s", string(fieldname)))
if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 {
parsedName := newName[1 : len(newName)-1]
log.Printf("Parse name: %s", parsedName)
fileField = parsedName
curParam := WorkflowAppActionParameter{
Name: "file_id",
Description: "Files to be uploaded",
Multiline: false,
Required: true,
Schema: SchemaDefinition{
Type: "string",
},
}
action.Parameters = append(action.Parameters, curParam)
}
}
}
}
}
}
headersFound := []string{}
if len(path.Post.Parameters) > 0 {
for counter, param := range path.Post.Parameters {
@@ -1610,12 +1677,17 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField)
if len(functionname) > 0 {
action.Name = functionname
}
//log.Printf("PARAMS: %d", len(action.Parameters))
//for _, param := range action.Parameters {
// log.Printf("%#v", param)
//}
return action, curCode
}
@@ -1743,7 +1815,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1877,7 +1949,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
+192 -9
View File
@@ -8,6 +8,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
@@ -125,12 +126,29 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri
return err
}
//log.Printf("FILENAME: %s", filename)
readFile, err := ioutil.ReadAll(fileReader)
if err != nil {
log.Printf("Not file: %s", err)
return err
}
// Fixes issues with older versions of Docker and reference formats
// Specific to Shuffle rn. Could expand.
// FIXME: Seems like the issue was with multi-stage builds
/*
if filename == "Dockerfile" {
log.Printf("Should search and replace in readfile.")
referenceCheck := "FROM frikky/shuffle:"
if strings.Contains(string(readFile), referenceCheck) {
log.Printf("SHOULD SEARCH & REPLACE!")
newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:")
readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1))
}
}
*/
//log.Printf("Filename: %s", filename)
// FIXME - might need the folder from EXTRA here
// Name has to be e.g. just "requirements.txt"
@@ -156,8 +174,23 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri
return nil
}
/*
// Fixes App SDK issues.. meh
func fixTags(tags []string) []string {
checkTag := "frikky/shuffle"
newTags := []string{}
for _, tag := range tags {
if strings.HasPrefix(tag, checkTags) {
newTags.append(newTags, fmt.Sprintf("registry.hub.docker.com/%s", tag))
}
newTags.append(tag)
}
}
*/
// Custom Docker image builder wrapper in memory
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error {
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
ctx := context.Background()
client, err := client.NewEnvClient()
if err != nil {
@@ -169,7 +202,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
tw := tar.NewWriter(buf)
defer tw.Close()
log.Printf("Setting up memory build structure for folder: %s", dockerfileFolder)
log.Printf("[INFO] Setting up memory build structure for folder: %s", dockerfileFolder)
err = getParsedTarMemory(fs, tw, dockerfileFolder, "")
if err != nil {
log.Printf("Tar issue: %s", err)
@@ -197,17 +230,56 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
}
// Build the actual image
log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder)
imageBuildResponse, err := client.ImageBuild(
ctx,
dockerFileTarReader,
buildOptions,
)
//log.Printf("Response: %#v", imageBuildResponse.Body)
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
defer imageBuildResponse.Body.Close()
_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
buildBuf := new(strings.Builder)
_, newerr := io.Copy(buildBuf, imageBuildResponse.Body)
if newerr != nil {
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
} else {
log.Printf("STRING: %s", buildBuf.String())
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
// Handles pulling of the same image if applicable
// This fixes some issues with older versions of Docker which can't build
// on their own ( <17.05 )
pullOptions := types.ImagePullOptions{}
downloaded := false
for _, image := range tags {
// Is this ok? Not sure. Tags shouldn't be controlled here prolly.
image = strings.ToLower(image)
newImage := fmt.Sprintf("%s/%s", registryName, image)
log.Printf("[INFO] Pulling image %s", newImage)
reader, err := client.ImagePull(ctx, newImage, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", newImage, err)
continue
}
// Attempt to retag the image to not contain registry...
//newBuf := buildBuf
downloaded = true
io.Copy(os.Stdout, reader)
log.Printf("[INFO] Successfully downloaded and built %s", newImage)
}
if !downloaded {
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
}
//baseDockerName
}
}
if err != nil {
@@ -272,9 +344,15 @@ func buildImage(tags []string, dockerfileFolder string) error {
// Read the STDOUT from the build process
defer imageBuildResponse.Body.Close()
_, err = io.Copy(os.Stdout, imageBuildResponse.Body)
buildBuf := new(strings.Builder)
_, err = io.Copy(buildBuf, imageBuildResponse.Body)
if err != nil {
return err
} else {
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ",")))
}
}
return nil
@@ -424,7 +502,7 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
hook, err := getHook(ctx, fileId)
if err != nil {
log.Printf("Failed getting hook: %s", err)
log.Printf("Failed getting hook %s (stop docker): %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -556,7 +634,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
hook, err := getHook(ctx, fileId)
if err != nil {
log.Printf("Failed getting hook: %s", err)
log.Printf("Failed getting hook %s (start docker): %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -636,7 +714,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
}
// FIXME - get some real data?
log.Printf("Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath)
log.Printf("[INFO] Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "message": "Started webhook"}`))
return
@@ -644,7 +722,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
// Checks if an image exists
func imageCheckBuilder(images []string) error {
log.Printf("[FIXME] ImageNames to check: %#v", images)
//log.Printf("[FIXME] ImageNames to check: %#v", images)
return nil
ctx := context.Background()
@@ -704,10 +782,115 @@ func hookTest() {
returnHook, err := getHook(ctx, hook.Id)
if err != nil {
log.Printf("Failed getting hook: %s", err)
log.Printf("Failed getting hook %s (test): %s", hook.Id, err)
}
if len(returnHook.Id) > 0 {
log.Printf("Success! - %s", returnHook.Id)
}
}
//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
func getDockerImage(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
_, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in validate swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type requestCheck struct {
Name string `datastore:"name" json:"name" yaml:"name"`
}
//body = []byte(`swagger: "2.0"`)
//body = []byte(`swagger: '1.0'`)
//newbody := string(body)
//newbody = strings.TrimSpace(newbody)
//body = []byte(newbody)
//log.Println(string(body))
//tmpbody, err := yaml.YAMLToJSON(body)
//log.Println(err)
//log.Println(string(tmpbody))
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version requestCheck
err = json.Unmarshal(body, &version)
if err != nil {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
}
log.Printf("Image to load: %s", version.Name)
//cli, err := client.NewEnvClient()
//if err != nil {
// log.Println("Unable to create docker client")
// return err
//}
dockercli, err := client.NewEnvClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
}
ctx := context.Background()
images, err := dockercli.ImageList(ctx, types.ImageListOptions{
All: true,
})
img := types.ImageSummary{}
tagFound := ""
for _, image := range images {
for _, tag := range image.RepoTags {
log.Printf("Image: %s", tag)
if strings.ToLower(tag) == strings.ToLower(version.Name) {
img = image
tagFound = tag
break
}
}
}
if len(img.ID) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
return
}
_ = tagFound
/*
log.Printf("IMg: %#v", img)
pullOptions := types.ImagePullOptions{}
log.Printf("[INFO] Pulling image %s", image)
reader, err := dockercli.ImagePull(ctx, tag, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", image, err)
}
io.Copy(os.Stdout, r)
*/
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name)))
}
+112 -3
View File
@@ -42,6 +42,9 @@ type File struct {
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"`
FileSize int64 `json:"filesize" datastore:"filesize"`
Duplicate bool `json:"duplicate" datastore:"duplicate"`
Subflows []string `json:"subflows" datastore:"subflows"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -100,6 +103,51 @@ func fileExists(filename string) bool {
return !info.IsDir()
}
func handleGetFiles(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUTH] User isn't admin")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`)))
return
}
ctx := context.Background()
files, err := getAllFiles(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed to get files: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`)))
return
}
log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id)
newBody, err := json.Marshal(files)
if err != nil {
log.Printf("[ERROR] Failed marshaling files: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -347,7 +395,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
@@ -417,8 +465,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
file.Status = "deleted"
err = setFile(ctx, *file)
if err != nil {
log.Printf("Failed setting file to uploading")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
return
}
//File not found, send 404
http.Error(resp, "File not found.", 404)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`))
return
}
@@ -552,6 +610,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
file.FileSize = int64(len(contents))
md5 := md5sum(contents)
buf.Reset()
@@ -642,7 +701,8 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
// Loads of validation below
if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
log.Printf("[ERROR] Missing field during upload.")
log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id")
log.Printf("INPUT: %s", string(body))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
return
@@ -715,6 +775,30 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
fileId := uuid.NewV4().String()
downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
duplicateWorkflows := []string{}
for _, trigger := range workflow.Triggers {
if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" {
for _, parameter := range trigger.Parameters {
if parameter.Name == "workflow" && len(parameter.Value) > 0 {
found := false
for _, workflow := range duplicateWorkflows {
if workflow == parameter.Value {
found = true
break
}
}
if !found {
duplicateWorkflows = append(duplicateWorkflows, parameter.Value)
}
break
}
}
}
}
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
@@ -726,6 +810,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
OrgId: curfile.OrgId,
WorkflowId: curfile.WorkflowId,
DownloadPath: downloadPath,
Subflows: duplicateWorkflows,
}
err = setFile(ctx, newFile)
@@ -740,6 +825,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
}
func getFile(ctx context.Context, id string) (*File, error) {
@@ -754,6 +840,9 @@ func getFile(ctx context.Context, id string) (*File, error) {
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
timeNow := time.Now().Unix()
file.UpdatedAt = timeNow
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
@@ -762,3 +851,23 @@ func setFile(ctx context.Context, file File) error {
return nil
}
func getAllFiles(ctx context.Context, orgId string) ([]File, error) {
var files []File
q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100)
_, err := dbclient.GetAll(ctx, q, &files)
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
q = q.Limit(50)
_, err := dbclient.GetAll(ctx, q, &files)
if err != nil {
return []File{}, err
}
} else {
return []File{}, err
}
}
return files, nil
}
+1
View File
@@ -23,6 +23,7 @@ require (
github.com/gorilla/mux v1.7.4
github.com/h2non/filetype v1.0.12
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/satori/go.uuid v1.2.0
golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+2
View File
@@ -160,6 +160,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+494 -115
View File
File diff suppressed because it is too large Load Diff
+1277 -324
View File
File diff suppressed because it is too large Load Diff
+25 -4
View File
@@ -1,5 +1,26 @@
#!/bin/sh
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26"
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
+6 -5
View File
@@ -2,7 +2,7 @@ version: '3'
services:
frontend:
#build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:0.8.3
image: ghcr.io/frikky/shuffle-frontend:0.8.56
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -17,7 +17,7 @@ services:
- backend
backend:
#build: ./backend
image: ghcr.io/frikky/shuffle-backend:0.8.3
image: ghcr.io/frikky/shuffle-backend:0.8.56
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
@@ -45,7 +45,7 @@ services:
- database
orborus:
#build: ./functions/onprem/orborus
image: ghcr.io/frikky/shuffle-orborus:0.8.0
image: ghcr.io/frikky/shuffle-orborus:0.8.5
container_name: shuffle-orborus
hostname: shuffle-orborus
networks:
@@ -53,8 +53,8 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- SHUFFLE_APP_SDK_VERSION=0.8.0
- SHUFFLE_WORKER_VERSION=0.8.0
- SHUFFLE_APP_SDK_VERSION=0.8.51
- SHUFFLE_WORKER_VERSION=0.8.54
- ORG_ID=${ORG_ID}
- ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
- BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
@@ -66,6 +66,7 @@ services:
- SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
- SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
- SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
- CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP}
restart: unless-stopped
database:
#build: ./backend/database
+4 -3
View File
@@ -8,7 +8,8 @@ ENV PATH /usr/src/app/node_modules/.bin:$PATH
COPY package.json /usr/src/app/package.json
RUN npm install --verbose
#RUN npm install --verbose
RUN yarn install
# copy only required files to not trigger rebuilding every time
COPY ./certs /usr/src/app/certs/
@@ -19,9 +20,9 @@ COPY ./*.json /usr/src/app/
# There were issues with the webpack installer from package.json
RUN rm -rf /usr/src/app/node_modules/webpack
RUN npm install webpack@4.42.0
#RUN yarn add webpack@4.42.0
RUN npm run-script build
RUN yarn build
# Production environment
FROM nginx:latest
+927 -116
View File
File diff suppressed because it is too large Load Diff
+4 -4
View File
@@ -1,13 +1,13 @@
{
"name": "shuffler",
"homepage": "https://shuffler.io",
"version": "0.6.0",
"version": "0.8.53",
"private": true,
"dependencies": {
"@material-ui/core": "^4.5.2",
"@material-ui/icons": "^4.5.1",
"@material-ui/styles": "^4.5.2",
"@use-it/interval": "^0.1.3",
"@use-it/interval": "^1.0.0",
"babel-eslint": "^10.1.0",
"class-transformer": "^0.3.1",
"create-react-app": "^2.0.3",
@@ -32,7 +32,7 @@
"md5-file": "^4.0.0",
"mdbreact": "^4.21.1",
"moment": "~2.20.1",
"react": "^16.10.2",
"react": "^16.14.0",
"react-alert": "^5.5.0",
"react-alert-template-basic": "^1.0.0",
"react-beforeunload": "^2.2.1",
@@ -40,7 +40,7 @@
"react-cookie": "^4.0.1",
"react-cytoscapejs": "^1.2.0",
"react-device-detect": "^1.9.10",
"react-dom": "^16.10.2",
"react-dom": "^16.14.0",
"react-draggable": "^3.3.2",
"react-dropzone": "^10.1.10",
"react-ga": "^2.7.0",
+1 -1
View File
@@ -142,7 +142,7 @@ const App = (message, props) => {
<Route exact path="/apps/new" render={props => <AppCreator isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} {...props} />} />
<Route exact path="/apps/edit/:appid" render={props => <AppCreator isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} {...props} />} />
<Route exact path="/schedules/:key" render={props => <EditSchedule globalUrl={globalUrl} {...props} />} />
<Route exact path="/workflows" render={props => <Workflows cookies={cookies} removeCookie={removeCookie} isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} cookies={cookies} {...props} />} />
<Route exact path="/workflows" render={props => <Workflows cookies={cookies} removeCookie={removeCookie} isLoaded={isLoaded} isLoggedIn={isLoggedIn} globalUrl={globalUrl} cookies={cookies} userdata={userdata} {...props} />} />
<Route exact path="/workflows/:key" render={props => <AngularWorkflow userdata={userdata} globalUrl={globalUrl} isLoaded={isLoaded} isLoggedIn={isLoggedIn} {...props} />} />
<Route exact path="/docs/:key" render={props => <Docs isLoaded={isLoaded} globalUrl={globalUrl} {...props} />} />
<Route exact path="/docs" render={props => { window.location.pathname = "/docs/about" }} />
Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

+166 -6
View File
@@ -3,9 +3,15 @@ import { makeStyles } from '@material-ui/styles';
import { useTheme } from '@material-ui/core/styles';
import Tooltip from '@material-ui/core/Tooltip';
import Grid from '@material-ui/core/Grid';
import Button from '@material-ui/core/Button';
import TextField from '@material-ui/core/TextField';
import Typography from '@material-ui/core/Typography';
import { useAlert } from "react-alert";
import IconButton from '@material-ui/core/IconButton';
import ExpandLessIcon from '@material-ui/icons/ExpandLess';
import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
import SaveIcon from '@material-ui/icons/Save';
const useStyles = makeStyles({
notchedOutline: {
@@ -23,11 +29,17 @@ const OrgHeader = (props) => {
const classes = useStyles()
var upload = ""
const defaultBranch = "master"
const [orgName, setOrgName] = React.useState(selectedOrganization.name)
const [orgDescription, setOrgDescription] = React.useState(selectedOrganization.description)
const [appDownloadUrl, setAppDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo === undefined || selectedOrganization.defaults.app_download_repo.length === 0 ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.app_download_repo)
const [appDownloadBranch, setAppDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.app_download_branch === undefined || selectedOrganization.defaults.app_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.app_download_branch)
const [workflowDownloadUrl, setWorkflowDownloadUrl] = React.useState(selectedOrganization.defaults === undefined ? "https://github.com/frikky/shuffle-apps" : selectedOrganization.defaults.workflow_download_repo === undefined || selectedOrganization.defaults.workflow_download_repo.length === 0 ? "https://github.com/frikky/shuffle-workflows" : selectedOrganization.defaults.workflow_download_repo)
const [workflowDownloadBranch, setWorkflowDownloadBranch] = React.useState(selectedOrganization.defaults === undefined ? defaultBranch : selectedOrganization.defaults.workflow_download_branch === undefined || selectedOrganization.defaults.workflow_download_branch.length === 0 ? defaultBranch : selectedOrganization.defaults.workflow_download_branch)
const [file, setFile] = React.useState("")
const [fileBase64, setFileBase64] = React.useState(selectedOrganization.image)
const [expanded, setExpanded] = React.useState(false)
if (file !== "") {
const img = document.getElementById('logo')
@@ -55,12 +67,13 @@ const OrgHeader = (props) => {
}
}
const handleEditOrg = (name, description, orgId, image) => {
const handleEditOrg = (name, description, orgId, image, defaults) => {
const data = {
"name": name,
"description": description,
"org_id": orgId,
"image": image,
"defaults": defaults,
}
const url = globalUrl + `/api/v1/orgs/${selectedOrganization.id}`;
@@ -102,9 +115,14 @@ const OrgHeader = (props) => {
style={{ width: 150, height: 55, flex: 1 }}
variant="contained"
color="primary"
onClick={() => handleEditOrg(orgName, orgDescription, selectedOrganization.id, selectedOrganization.image)}
onClick={() => handleEditOrg(orgName, orgDescription, selectedOrganization.id, selectedOrganization.image, {
"app_download_repo": appDownloadUrl,
"app_download_branch": appDownloadBranch,
"workflow_download_repo": workflowDownloadUrl,
"workflow_download_branch": workflowDownloadBranch,
})}
>
Save Changes
<SaveIcon />
</Button>
var imageData = file.length > 0 ? file : fileBase64
@@ -113,7 +131,7 @@ const OrgHeader = (props) => {
return (
<div>
<div style={{color: "white", flex: "1", display: "flex", flexDirection: "row"}}>
<Tooltip title="Click to edit the app's image - 174x174" placement="bottom">
<Tooltip title="Click to edit the organizations's image (174x174)" placement="bottom">
<div style={{flex: "1", margin: "10px 25px 10px 0px", border: imageData !== undefined && imageData.length > 0 ? null : "1px solid #f85a3e", cursor: "pointer", backgroundColor: imageData !== undefined && imageData.length > 0 ? null : theme.palette.inputColor, maxWidth: 174, maxHeight: 174}} onClick={() => {upload.click()}}>
<input hidden type="file" ref={(ref) => upload = ref} onChange={editHeaderImage} />
{imageInfo}
@@ -188,9 +206,151 @@ const OrgHeader = (props) => {
<div style={{margin: "auto", textalign: "center",}}>
{orgSaveButton}
</div>
</div>
</div>
</div>
</div>
</div>
<div style={{textAlign: "center",}}>
<IconButton style={{color: "white", marginTop: 10, }} onClick={() => {
setExpanded(!expanded)
}}>
{expanded ?
<ExpandLessIcon />
:
<ExpandMoreIcon />
}
</IconButton>
{expanded ?
<Grid container spacing={3} style={{textAlign: "left"}}>
<Grid item xs={6} style={{}}>
<span>
<Typography>
App Download URL
</Typography>
<TextField
required
style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
fullWidth={true}
type="name"
id="outlined-with-placeholder"
margin="normal"
variant="outlined"
placeholder="A description for the organization"
value={appDownloadUrl}
onChange={e => {
setAppDownloadUrl(e.target.value)
}}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
},
style:{
color: "white",
},
}}
/>
</span>
</Grid>
<Grid item xs={6} style={{}}>
<span>
<Typography>
App Download Branch
</Typography>
<TextField
required
style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
fullWidth={true}
type="name"
id="outlined-with-placeholder"
margin="normal"
variant="outlined"
placeholder="A description for the organization"
value={appDownloadBranch}
onChange={e => {
setAppDownloadBranch(e.target.value)
}}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
},
style:{
color: "white",
},
}}
/>
</span>
</Grid>
<Grid item xs={6} style={{}}>
<span>
<Typography>
Workflow Download URL
</Typography>
<TextField
required
style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
fullWidth={true}
type="name"
id="outlined-with-placeholder"
margin="normal"
variant="outlined"
placeholder="A description for the organization"
value={workflowDownloadUrl}
onChange={e => {
setWorkflowDownloadUrl(e.target.value)
}}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
},
style:{
color: "white",
},
}}
/>
</span>
</Grid>
<Grid item xs={6} style={{}}>
<span>
<Typography>
Workflow Download Branch
</Typography>
<TextField
required
style={{flex: "1", marginTop: "5px", marginRight: "15px", backgroundColor: theme.palette.inputColor}}
fullWidth={true}
type="name"
id="outlined-with-placeholder"
margin="normal"
variant="outlined"
placeholder="A description for the organization"
value={workflowDownloadBranch}
onChange={e => {
setWorkflowDownloadBranch(e.target.value)
}}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
},
style:{
color: "white",
},
}}
/>
</span>
</Grid>
{/*
<span style={{textAlign: "center"}}>
{expanded ?
<ExpandLessIcon />
:
<ExpandMoreIcon />
}
</span>
*/}
</Grid>
:
null
}
</div>
</div>
)
}
+2
View File
@@ -74,6 +74,8 @@ const data = [{
'shape': 'octagon',
'border-color': 'orange',
'background-color': '#213243',
'background-width': '100%',
'background-height': '100%',
},
},
{
+489 -119
View File
@@ -1,4 +1,4 @@
import React, { useEffect} from 'react';
import React, { useState } from 'react';
import { makeStyles } from '@material-ui/styles';
import {Link} from 'react-router-dom';
@@ -31,6 +31,11 @@ import { useTheme } from '@material-ui/core/styles';
import HandlePayment from './HandlePayment'
import OrgHeader from '../components/OrgHeader'
import EditIcon from '@material-ui/icons/Edit';
import SelectAllIcon from '@material-ui/icons/SelectAll';
import OpenInNewIcon from '@material-ui/icons/OpenInNew';
import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
import DescriptionIcon from '@material-ui/icons/Description';
import PolymerIcon from '@material-ui/icons/Polymer';
import CheckCircleIcon from '@material-ui/icons/CheckCircle';
import CloseIcon from '@material-ui/icons/Close';
@@ -59,6 +64,7 @@ const Admin = (props) => {
const theme = useTheme();
const classes = useStyles();
const [firstRequest, setFirstRequest] = React.useState(true);
const [orgRequest, setOrgRequest] = React.useState(true);
const [modalUser, setModalUser] = React.useState({});
const [modalOpen, setModalOpen] = React.useState(false);
@@ -78,11 +84,13 @@ const Admin = (props) => {
const [environments, setEnvironments] = React.useState([]);
const [authentication, setAuthentication] = React.useState([]);
const [schedules, setSchedules] = React.useState([])
const [files, setFiles] = React.useState([])
const [selectedUser, setSelectedUser] = React.useState({})
const [newPassword, setNewPassword] = React.useState("");
const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false)
const [selectedAuthentication, setSelectedAuthentication] = React.useState({})
const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false)
const [authenticationFields, setAuthenticationFields] = React.useState([])
const [showArchived, setShowArchived] = React.useState(false)
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"
@@ -272,9 +280,70 @@ const Admin = (props) => {
})
}
const saveAuthentication = (authentication) => {
const data = authentication
const url = globalUrl + '/api/v1/apps/authentication';
fetch(url, {
mode: 'cors',
method: 'PUT',
body: JSON.stringify(data),
credentials: 'include',
crossDomain: true,
withCredentials: true,
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
})
.then(response =>
response.json().then(responseJson => {
if (responseJson["success"] === false) {
alert.error("Failed changing authentication")
} else {
//alert.success("Successfully password!")
setSelectedUserModalOpen(false)
getAppAuthentication()
}
}),
)
.catch(error => {
alert.error("Err: " + error.toString())
});
}
const editAuthenticationConfig = (id) => {
const data = {
"id": id,
"action": "assign_everywhere",
}
const url = globalUrl + '/api/v1/apps/authentication/'+id+"/config";
fetch(url, {
mode: 'cors',
method: 'POST',
body: JSON.stringify(data),
credentials: 'include',
crossDomain: true,
withCredentials: true,
headers: {
'Content-Type': 'application/json; charset=utf-8',
},
})
.then(response =>
response.json().then(responseJson => {
if (responseJson["success"] === false) {
alert.error("Failed overwriting appauth in workflows")
} else {
alert.success("Successfully updated auth everywhere!")
setSelectedUserModalOpen(false)
getAppAuthentication()
}
}),
)
.catch(error => {
alert.error("Err: " + error.toString())
});
}
const onPasswordChange = () => {
const data = { "username": selectedUser.username, "newpassword": newPassword }
@@ -296,7 +365,7 @@ const Admin = (props) => {
if (responseJson["success"] === false) {
alert.error("Failed setting new password")
} else {
alert.success("Successfully password!")
alert.success("Successfully updated password!")
setSelectedUserModalOpen(false)
}
}),
@@ -466,6 +535,33 @@ const Admin = (props) => {
})
}
const flushQueue = (name) => {
// Just use this one?
const url = globalUrl + '/api/v1/flush_queue';
fetch(url, {
method: 'DELETE',
credentials: "include",
headers: {
'Content-Type': 'application/json',
},
})
.then(response =>
response.json().then(responseJson => {
if (responseJson["success"] === false) {
alert.error(responseJson.reason)
getEnvironments()
} else {
setLoginInfo("")
setModalOpen(false)
getEnvironments()
}
}),
)
.catch(error => {
console.log("Error when deleting: ", error)
})
}
const deleteEnvironment = (name) => {
// FIXME - add some check here ROFL
alert.info("Deleting environment " + name)
@@ -550,6 +646,78 @@ const Admin = (props) => {
});
}
const getFiles = () => {
fetch(globalUrl + "/api/v1/files", {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
return
}
return response.json()
})
.then((responseJson) => {
//console.log(responseJson)
setFiles(responseJson)
})
.catch(error => {
alert.error(error.toString())
});
}
const downloadFile = (file) => {
fetch(globalUrl + "/api/v1/files/"+file.id+"/content", {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for apps :O!")
return ""
}
return response.text()
})
.then((respdata) => {
if (respdata.length === 0) {
alert.error("Failed getting file")
return
}
var blob = new Blob( [ respdata ], {
type: 'application/octet-stream'
})
var url = URL.createObjectURL( blob )
var link = document.createElement( 'a' )
link.setAttribute( 'href', url )
link.setAttribute( 'download', `${file.filename}` )
var event = document.createEvent( 'MouseEvents' )
event.initMouseEvent( 'click', true, true, window, 1, 0, 0, 0, 0, false, false, false, false, 0, null)
link.dispatchEvent( event )
//return response.json()
})
.then((responseJson) => {
//console.log(responseJson)
//setSchedules(responseJson)
})
.catch(error => {
alert.error(error.toString())
});
}
const getSchedules = () => {
fetch(globalUrl + "/api/v1/workflows/schedules", {
method: 'GET',
@@ -596,6 +764,7 @@ const Admin = (props) => {
.then((responseJson) => {
if (responseJson.success) {
//console.log(responseJson.data)
console.log(responseJson)
setAuthentication(responseJson.data)
} else {
alert.error("Failed getting authentications")
@@ -705,6 +874,48 @@ const Admin = (props) => {
});
}
const setConfig = (event, newValue) => {
if (newValue === 1) {
getUsers()
} else if (newValue === 2) {
getAppAuthentication()
} else if (newValue === 3) {
getEnvironments()
} else if (newValue === 4) {
getSchedules()
} else if (newValue === 5) {
getFiles()
} else if (newValue === 6) {
getOrgs()
}
if (newValue === 6) {
console.log("Should get apps for categories.")
}
const views = {
0: "organization",
1: "users",
2: "app_auth",
3: "environments",
4: "schedules",
5: "files",
6: "categories",
}
//var theURL = window.location.pathname
//FIXME: Add url edits
//var theURL = window.location
//theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`)
//window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath);
//console.log(newpath)
//window.location.pathame = newpath
setModalUser({})
setCurTab(newValue)
}
if (firstRequest) {
setFirstRequest(false)
@@ -720,19 +931,20 @@ const Admin = (props) => {
"app_auth": 2,
"environments": 3,
"schedules": 4,
"categories": 5,
"files": 5,
}
if (props.match.params.key !== undefined) {
const tmpitem = views[props.match.params.key]
if (tmpitem !== undefined) {
setCurTab(tmpitem)
//setCurTab(tmpitem)
setConfig("", tmpitem)
}
}
}
if (selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined) {
//setSelectedOrganization(userdata.active_org)
if (selectedOrganization.id === undefined && userdata !== undefined && userdata.active_org !== undefined && orgRequest) {
setOrgRequest(false)
handleGetOrg(userdata.active_org.id)
}
@@ -820,8 +1032,8 @@ const Admin = (props) => {
});
}
const editAuthenticationModal =
<Dialog modal
const editAuthenticationModal = selectedAuthenticationModalOpen ?
<Dialog
open={selectedAuthenticationModalOpen}
onClose={() => { setSelectedAuthenticationModalOpen(false) }}
PaperProps={{
@@ -833,56 +1045,71 @@ const Admin = (props) => {
},
}}
>
<DialogTitle><span style={{ color: "white" }}>Edit authentication</span></DialogTitle>
<DialogTitle><span style={{ color: "white" }}>Edit authentication for {selectedAuthentication.app.name} ({selectedAuthentication.label})</span></DialogTitle>
<DialogContent>
<div style={{ display: "flex" }}>
<TextField
style={{ backgroundColor: theme.palette.inputColor, flex: 3 }}
InputProps={{
style: {
height: 50,
color: "white",
},
}}
color="primary"
required
fullWidth={true}
placeholder="New password"
type="password"
id="standard-required"
autoComplete="password"
margin="normal"
variant="outlined"
onChange={e => setNewPassword(e.target.value)}
/>
<Button
style={{ maxHeight: 50, flex: 1 }}
variant="outlined"
color="primary"
onClick={() => onPasswordChange()}
>
Submit
</Button>
</div>
<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
<Button
style={{}}
variant="outlined"
color="primary"
onClick={() => deleteUser(selectedUser)}
>
{selectedUser.active ? "Deactivate" : "Activate"}
</Button>
<Button
style={{}}
variant="outlined"
color="primary"
onClick={() => generateApikey(selectedUser.id)}
>
Get new API key
</Button>
{selectedAuthentication.fields.map((data, index) => {
return (
<div key={index}>
<Typography style={{marginBottom: 0, marginTop: 10}}>{data.key}</Typography>
<TextField
style={{ backgroundColor: theme.palette.inputColor, marginTop: 0, }}
InputProps={{
style: {
height: 50,
color: "white",
},
}}
color="primary"
required
fullWidth={true}
placeholder={data.key}
type="text"
id={`authentication-${index}`}
margin="normal"
variant="outlined"
onChange={e => {
authenticationFields[index].value = e.target.value
setAuthenticationFields(authenticationFields)
}}
/>
</div>
)
})}
</DialogContent>
<DialogActions>
<Button style={{ borderRadius: "0px" }} onClick={() => setSelectedAuthenticationModalOpen(false)} color="primary">
Cancel
</Button>
<Button variant="contained" style={{ borderRadius: "0px" }} onClick={() => {
var error = false
for (var key in authenticationFields) {
const item = authenticationFields[key]
if (item.value.length === 0) {
console.log("ITEM: ", item)
//var currentnode = cy.getElementById(data.id)
var textfield = document.getElementById(`authentication-${key}`)
if (textfield !== null && textfield !== undefined) {
console.log("HANDLE ERROR FOR KEY ", key)
}
error = true
}
}
if (error) {
alert.error("All fields must have a new value")
} else {
alert.success("Saving new version of this authentication")
selectedAuthentication.fields = authenticationFields
saveAuthentication(selectedAuthentication)
setSelectedAuthentication({})
setSelectedAuthenticationModalOpen(false)
}
}} color="primary">
Submit
</Button>
</DialogActions>
</Dialog>
: null
const editUserModal =
<Dialog modal
@@ -897,7 +1124,7 @@ const Admin = (props) => {
},
}}
>
<DialogTitle><span style={{ color: "white" }}>Edit user</span></DialogTitle>
<DialogTitle><span style={{ color: "white" }}><EditIcon /></span></DialogTitle>
<DialogContent>
<div style={{ display: "flex" }}>
<TextField
@@ -1270,7 +1497,7 @@ const Admin = (props) => {
})}
</Grid>
<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
{isCloud && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ?
{isCloud && selectedOrganization.subscriptions !== undefined && selectedOrganization.subscriptions !== null && selectedOrganization.subscriptions.length > 0 ?
<div style={{marginTop: 30, marginBottom: 20}}>
<Typography style={{marginTop: 40, marginLeft: 10, marginBottom: 5,}}>
Your subscription{selectedOrganization.subscriptions.length > 1 ? "s" : ""}
@@ -1429,6 +1656,7 @@ const Admin = (props) => {
</div>
<div />
<Button
disabled={isCloud}
style={{}}
variant="contained"
color="primary"
@@ -1463,8 +1691,13 @@ const Admin = (props) => {
/>
</ListItem>
{users === undefined ? null : users.map((data, index) => {
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
}
return (
<ListItem key={index}>
<ListItem key={index} style={{backgroundColor: bgColor}}>
<ListItemText
primary={data.username}
style={{ minWidth: 200, maxWidth: 200 }}
@@ -1532,6 +1765,113 @@ const Admin = (props) => {
</div>
: null
const filesView = curTab === 5 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Files</h2>
<span style={{marginLeft: 25}}>Files from Workflows. <a target="_blank" href="https://shuffler.io/docs/organizations#files" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a></span>
</div>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor}}/>
<List>
<ListItem>
<ListItemText
primary="Created"
style={{maxWidth: 225, minWidth: 225}}
/>
<ListItemText
primary="Name"
style={{maxWidth: 150, minWidth: 150, overflow: "hidden",}}
/>
<ListItemText
primary="Workflow"
style={{maxWidth: 100, minWidth: 100, overflow: "hidden",}}
/>
<ListItemText
primary="Md5"
style={{minWidth: 300, maxWidth: 300, overflow: "hidden"}}
/>
<ListItemText
primary="Status"
style={{minWidth: 75, maxWidth: 75}}
/>
<ListItemText
primary="Filesize"
style={{minWidth: 125, maxWidth: 125}}
/>
<ListItemText
primary="Actions"
/>
</ListItem>
{files === undefined || files === null ? null : files.map((file, index) => {
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
}
return (
<ListItem key={index} style={{backgroundColor: bgColor}} >
<ListItemText
style={{maxWidth: 225, minWidth: 225}}
primary={new Date(file.created_at*1000).toISOString()}
/>
<ListItemText
style={{maxWidth: 150, minWidth: 150}}
primary={file.filename}
/>
<ListItemText
primary=
<Tooltip title={"Go to workflow"} style={{}} aria-label={"Download"}>
<a style={{textDecoration: "none", color: "#f85a3e"}} href={`/workflows/${file.workflow_id}`} target="_blank">
<IconButton>
<OpenInNewIcon style={{color: "white"}} />
</IconButton>
</a>
</Tooltip>
style={{minWidth: 100, maxWidth: 100, overflow: "hidden"}}
/>
<ListItemText
primary={file.md5_sum}
style={{minWidth: 300, maxWidth: 300, overflow: "hidden"}}
/>
<ListItemText
primary={file.status}
style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}}
/>
<ListItemText
primary={file.filesize}
style={{minWidth: 125, maxWidth: 125, overflow: "hidden"}}
/>
<ListItemText
primary=
<Tooltip title={"Download file"} style={{}} aria-label={"Download"}>
<IconButton onClick={() => {
downloadFile(file)
}}>
<CloudDownloadIcon style={{color: "white"}} />
</IconButton>
</Tooltip>
style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}}
/>
{/*
<ListItemText>
<Button
style={{}}
variant="contained"
color="primary"
disabled
onClick={() => deleteSchedule(file)}
>
Stop schedule
</Button>
</ListItemText>
*/}
</ListItem>
)
})}
</List>
</div>
: null
const schedulesView = curTab === 4 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
@@ -1562,8 +1902,13 @@ const Admin = (props) => {
/>
</ListItem>
{schedules === undefined || schedules === null ? null : schedules.map((schedule, index) => {
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
}
return (
<ListItem key={index}>
<ListItem key={index} style={{backgroundColor: bgColor}}>
<ListItemText
style={{maxWidth: 200, minWidth: 200}}
primary={schedule.environment === "cloud" ? schedule.frequency : <span>{schedule.seconds} seconds</span>}
@@ -1673,35 +2018,53 @@ const Admin = (props) => {
</div>
: null
const updateAppAuthentication = (field) => {
setSelectedAuthenticationModalOpen(true)
setSelectedAuthentication(field)
//{selectedAuthentication.fields.map((data, index) => {
var newfields = []
for (var key in field.fields) {
newfields.push({
"key": field.fields[key].key,
"value": "",
})
}
setAuthenticationFields(newfields)
}
const authenticationView = curTab === 2 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>App Authentication</h2>
<span style={{marginLeft: 25}}>Control the authentication options for individual apps. <b>Actions can be destructive!</b></span>
.&nbsp;<a target="_blank" href="https://shuffler.io/docs/organizations#app_authentication" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a>
&nbsp;<a target="_blank" href="https://shuffler.io/docs/organizations#app_authentication" style={{textDecoration: "none", color: "#f85a3e"}}>Learn more</a>
</div>
<Divider style={{marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor}}/>
<List>
<ListItem>
<ListItemText
primary="Icon"
style={{minWidth: 150, maxWidth: 150}}
style={{minWidth: 75, maxWidth: 75}}
/>
<ListItemText
primary="Label"
style={{minWidth: 250, maxWidth: 250}}
style={{minWidth: 225, maxWidth: 225}}
/>
<ListItemText
primary="App Name"
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary="Workflows"
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
primary="Ready"
style={{minWidth: 100, maxWidth: 100}}
/>
<ListItemText
primary="Action amount"
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
primary="Workflows"
style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
/>
<ListItemText
primary="Actions"
style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
/>
<ListItemText
primary="Fields"
@@ -1712,27 +2075,36 @@ const Admin = (props) => {
/>
</ListItem>
{authentication === undefined ? null : authentication.map((data, index) => {
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
}
return (
<ListItem key={index}>
<ListItem key={index} style={{backgroundColor: bgColor}}>
<ListItemText
primary=<img alt="" src={data.app.large_image} style={{maxWidth: 50,}} />
style={{minWidth: 150, maxWidth: 150}}
style={{minWidth: 75, maxWidth: 75}}
/>
<ListItemText
primary={data.label}
style={{minWidth: 250, maxWidth: 250}}
style={{minWidth: 225, maxWidth: 225}}
/>
<ListItemText
primary={data.app.name}
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.usage === null ? 0 : data.usage.length}
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
primary={data.defined === false ? "No" : "Yes"}
style={{minWidth: 100, maxWidth: 100}}
/>
<ListItemText
primary={data.workflow_count === null ? 0 : data.workflow_count}
style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
/>
<ListItemText
primary={data.node_count}
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
style={{minWidth: 110, maxWidth: 110, overflow: "hidden"}}
/>
<ListItemText
primary={data.fields.map(data => {
@@ -1741,16 +2113,43 @@ const Admin = (props) => {
style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}}
/>
<ListItemText>
<Button
style={{}}
variant="outlined"
color="primary"
<IconButton
onClick={() => {
updateAppAuthentication(data)
}}
>
<EditIcon color="primary"/>
</IconButton>
{data.defined ?
<Tooltip color="primary" title="Set in EVERY workflow" placement="top">
<IconButton
style={{marginRight: 10}}
disabled={data.defined === false}
onClick={() => {
editAuthenticationConfig(data.id)
}}
>
<SelectAllIcon color={data.defined ? "primary" : "secondary"} />
</IconButton>
</Tooltip>
:
<Tooltip color="primary" title="Must edit before you can set in all workflows" placement="top">
<IconButton
style={{marginRight: 10}}
onClick={() => {
}}
>
<SelectAllIcon color={data.defined ? "primary" : "secondary"} />
</IconButton>
</Tooltip>
}
<IconButton
onClick={() => {
deleteAuthentication(data)
}}
>
Delete
</Button>
<DeleteIcon color="primary"/>
</IconButton>
</ListItemText>
</ListItem>
)
@@ -1820,6 +2219,11 @@ const Admin = (props) => {
return null
}
//var bgColor = "#27292d"
//if (index % 2 === 0) {
// bgColor = "#1f2023"
//}
return (
<ListItem key={index}>
<ListItemText
@@ -1848,6 +2252,7 @@ const Admin = (props) => {
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
>
<Button disabled={environment.archived} variant="outlined" style={{borderRadius: "0px"}} onClick={() => deleteEnvironment(environment.Name)} color="primary">Archive</Button>
{/*<Button disabled={environment.archived} variant="outlined" style={{borderRadius: "0px"}} onClick={() => flushQueue(environment.Name)} color="primary">Flush Queue</Button>*/}
</ListItemText>
<ListItemText
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
@@ -1860,7 +2265,7 @@ const Admin = (props) => {
</div>
: null
const organizationsTab = curTab === 6 ?
const organizationsTab = curTab === 7 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Organizations</h2>
@@ -1941,7 +2346,7 @@ const Admin = (props) => {
</div>
: null
const hybridTab = curTab === 5 ?
const hybridTab = curTab === 6 ?
<div>
<div style={{marginTop: 20, marginBottom: 20,}}>
<h2 style={{display: "inline",}}>Hybrid</h2>
@@ -1983,48 +2388,11 @@ const Admin = (props) => {
// primary={environment.Registered ? "true" : "false"}
const setConfig = (event, newValue) => {
if (newValue === 1) {
getUsers()
} else if (newValue === 2) {
getAppAuthentication()
} else if (newValue === 3) {
getEnvironments()
} else if (newValue === 4) {
getSchedules()
} else if (newValue === 6) {
getOrgs()
}
if (newValue === 6) {
console.log("Should get apps for categories.")
}
const views = {
0: "organization",
1: "users",
2: "app_auth",
3: "environments",
4: "schedules",
5: "categories",
}
//var theURL = window.location.pathname
//FIXME: Add url edits
//var theURL = window.location
//theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`)
//window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath);
//console.log(newpath)
//window.location.pathame = newpath
setModalUser({})
setCurTab(newValue)
}
const iconStyle = {marginRight: 10}
const data =
<div style={{minWidth: 1366, margin: "auto"}}>
<div style={{width: 1366, margin: "auto", overflowX: "hidden",}}>
<Paper style={paperStyle}>
<Tabs
value={curTab}
@@ -2033,10 +2401,11 @@ const Admin = (props) => {
aria-label="disabled tabs example"
>
<Tab label=<span><BusinessIcon style={iconStyle} /> Organization</span>/>
{isCloud ? null : <Tab label=<span><AccessibilityNewIcon style={iconStyle} />Users</span> />}
<Tab label=<span><AccessibilityNewIcon style={iconStyle} />Users</span> />
{isCloud ? null : <Tab label=<span><LockIcon style={iconStyle} />App Authentication</span>/>}
{isCloud ? null : <Tab label=<span><EcoIcon style={iconStyle} />Environments</span>/>}
{isCloud ? null : <Tab label=<span><ScheduleIcon style={iconStyle} />Schedules</span> />}
{isCloud ? null : <Tab label=<span><DescriptionIcon style={iconStyle} />Files</span> />}
{window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><CloudIcon style={iconStyle} /> Hybrid</span>/> : null}
{window.location.protocol == "http:" && window.location.port === "3000" ? <Tab label=<span><BusinessIcon style={iconStyle} /> Organizations</span>/> : null}
{window.location.protocol === "http:" && window.location.port === "3000" ? <Tab label=<span><LockIcon style={iconStyle} />Categories</span>/> : null}
@@ -2049,6 +2418,7 @@ const Admin = (props) => {
{usersView}
{environmentView}
{schedulesView}
{filesView}
{hybridTab}
{organizationsTab}
</div>
+1 -6
View File
@@ -6,11 +6,6 @@ import TextField from '@material-ui/core/TextField';
import Button from '@material-ui/core/Button';
import Paper from '@material-ui/core/Paper';
const hrefStyle = {
color: "white",
textDecoration: "none"
}
const bodyDivStyle = {
margin: "auto",
marginTop: "100px",
@@ -35,7 +30,7 @@ const useStyles = makeStyles({
});
const AdminAccount = props => {
const { globalUrl, isLoaded, isLoggedIn, setIsLoggedIn, setCookie, } = props;
const { globalUrl, isLoaded, isLoggedIn, } = props;
const [username, setUsername] = useState("");
const [password, setPassword] = useState("");
File diff suppressed because one or more lines are too long
+492 -108
View File
@@ -4,6 +4,7 @@ import {BrowserView, MobileView} from "react-device-detect";
import {Link} from 'react-router-dom';
import Paper from '@material-ui/core/Paper';
import Typography from '@material-ui/core/Typography';
import FormControlLabel from '@material-ui/core/FormControlLabel';
import Button from '@material-ui/core/Button';
import Divider from '@material-ui/core/Divider';
@@ -18,6 +19,7 @@ import DialogActions from '@material-ui/core/DialogActions';
import TextField from '@material-ui/core/TextField';
import Tooltip from '@material-ui/core/Tooltip';
import CheckCircleIcon from '@material-ui/icons/CheckCircle';
import AttachFileIcon from '@material-ui/icons/AttachFile';
import Breadcrumbs from '@material-ui/core/Breadcrumbs';
import AppsIcon from '@material-ui/icons/Apps';
import CircularProgress from '@material-ui/core/CircularProgress';
@@ -25,6 +27,7 @@ import CircularProgress from '@material-ui/core/CircularProgress';
import Chip from '@material-ui/core/Chip';
import ChipInput from 'material-ui-chip-input'
import YAML from 'yaml'
import ErrorOutline from '@material-ui/icons/ErrorOutline';
import { useAlert } from "react-alert";
import words from "shellwords"
@@ -99,7 +102,12 @@ const parseCurl = (s) => {
return ""
}
var args = rewrite(words.split(s))
try {
var args = rewrite(words.split(s))
} catch (e) {
return s
}
var out = { method: 'GET', header: {} }
var state = ''
@@ -187,6 +195,7 @@ const AppCreator = (props) => {
const alert = useAlert()
var upload = ""
const increaseAmount = 30
const actionNonBodyRequest = ["GET", "HEAD", "DELETE", "CONNECT"]
const actionBodyRequest = ["POST", "PUT", "PATCH",]
const authenticationOptions = ["No authentication", "API key", "Bearer auth", "Basic auth", ]
@@ -218,6 +227,9 @@ const AppCreator = (props) => {
const [actions, setActions] = useState([])
const [errorCode, setErrorCode] = useState("")
const [appBuilding, setAppBuilding] = useState(false)
const [extraBodyFields, setExtraBodyFields] = useState([])
const [fileUploadEnabled, setFileUploadEnabled] = useState(false)
const [actionAmount, setActionAmount] = useState(increaseAmount)
//const [actions, setActions] = useState([{
// "name": "Get workflows",
@@ -246,6 +258,7 @@ const AppCreator = (props) => {
const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0])
const [currentAction, setCurrentAction] = useState({
"name": "",
"file_field": "",
"description": "",
"url": "",
"headers": "",
@@ -253,6 +266,7 @@ const AppCreator = (props) => {
"queries": [],
"body": "",
"errors": [],
"example_response": "",
"method": actionNonBodyRequest[0],
});
@@ -262,7 +276,7 @@ const AppCreator = (props) => {
if (firstrequest) {
setFirstrequest(false)
if (window.location.pathname.includes("apps/edit")) {
setIsEditing(true)
setIsEditing(true)
handleEditApp()
} else {
checkQuery()
@@ -322,15 +336,37 @@ const AppCreator = (props) => {
throw new Error("NOT 200 :O")
}
//console.log("DATA: ", response.text())
return response.json()
})
.then((responseJson) => {
console.log("THE BODY IS HERE")
setIsAppLoaded(true)
if (!responseJson.success) {
alert.error("Failed to verify")
} else {
const data = JSON.parse(responseJson.body)
parseIncomingOpenapiData(data)
} else{
console.log("HMM 2")
var jsonvalid = false
var tmpvalue = ""
try {
tmpvalue = JSON.parse(responseJson.body)
jsonvalid = true
} catch (e) {
console.log("Error JSON: ", e)
}
if (!jsonvalid) {
try {
tmpvalue = YAML.parse(responseJson.body, )
jsonvalid = true
} catch(e) {
console.log("Error YAML: ", e)
}
}
if (jsonvalid) {
parseIncomingOpenapiData(tmpvalue)
}
}
})
.catch(error => {
@@ -353,16 +389,52 @@ const AppCreator = (props) => {
//}
}
const handleGetRef = (parameter, data) => {
if (parameter["$ref"] === undefined) {
return parameter
}
const paramsplit = parameter["$ref"].split("/")
if (paramsplit[0] !== "#") {
console.log("Bad param: ", paramsplit)
return parameter
}
var newitem = data
for (var key in paramsplit) {
var tmpparam = paramsplit[key]
if (tmpparam === "#") {
continue
}
if (newitem[tmpparam] === undefined) {
return parameter
}
newitem = newitem[tmpparam]
}
return newitem
//console.log("Should get ", parameter["$ref"])
//const subkeys = parameter["$ref"].split("/")
// setBasedata(data)
// handleGetReference(parameter["$ref"])
}
// Sets the data up as it should be at later points
// This is the data FROM the database, not what's being saved
const parseIncomingOpenapiData = (data) => {
//console.log("DATA: ", data.info)
setBasedata(data)
setName(data.info.title)
setDescription(data.info.description)
document.title = "Apps - "+data.info.title
if (data.info !== null && data.info !== undefined) {
setName(data.info.title)
setDescription(data.info.description)
document.title = "Apps - "+data.info.title
if (data.info["x-logo"] !== undefined) {
setFileBase64(data.info["x-logo"])
}
@@ -377,11 +449,21 @@ const AppCreator = (props) => {
}
if (data.tags !== undefined && data.tags.length > 0) {
var newtags = []
for (var key in data.tags) {
newWorkflowTags.push(data.tags[key].name)
if (data.tags[key].name.length > 50) {
console.log("Skipping tag cus it's too long: ", data.tags[key].name.length)
continue
}
newtags.push(data.tags[key].name)
}
setNewWorkflowTags(newWorkflowTags)
if (newtags.length > 10) {
newtags = newtags.slice(0,9)
}
setNewWorkflowTags(newtags)
}
// This is annoying (:
@@ -410,16 +492,17 @@ const AppCreator = (props) => {
for (let [path, pathvalue] of Object.entries(data.paths)) {
for (let [method, methodvalue] of Object.entries(pathvalue)) {
if (methodvalue === null) {
alert.info("Skipped method "+method)
alert.info("Skipped method (null)"+method)
continue
}
if (!allowedfunctions.includes(method.toUpperCase())) {
alert.info("Skipped method (not allowed) "+method)
continue
}
var tmpname = methodvalue.summary
if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0) {
if (methodvalue.operationId !== undefined && methodvalue.operationId !== null && methodvalue.operationId.length > 0 && (tmpname === undefined || tmpname.length === 0)) {
tmpname = methodvalue.operationId
}
@@ -427,16 +510,94 @@ const AppCreator = (props) => {
"name": tmpname,
"description": methodvalue.description,
"url": path,
"file_field": "",
"method": method.toUpperCase(),
"headers": "",
"queries": [],
"paths": [],
"body": "",
"errors": [],
"example_response": "",
}
if (methodvalue["requestBody"] !== undefined) {
//console.log("Handle requestbody: ", methodvalue["requestBody"])
if (methodvalue["requestBody"]["content"] !== undefined) {
if (methodvalue["requestBody"]["content"]["application/json"] !== undefined) {
if (methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null) {
if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) {
var tmpobject = {}
for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"])) {
tmpobject[prop] = `\$\{${prop}\}`
}
for (var subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) {
const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey]
tmpobject[tmpitem] = `\$\{${tmpitem}\}`
}
newaction["body"] = JSON.stringify(tmpobject, null, 2)
}
}
} else if (methodvalue["requestBody"]["content"]["application/xml"] !== undefined) {
console.log("METHOD XML: ", methodvalue)
if (methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/xml"]["schema"] !== null) {
if (methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"] !== undefined) {
var tmpobject = {}
for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) {
tmpobject[prop] = `\$\{${prop}\}`
}
for (var subkey in methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"]) {
const tmpitem = methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"][subkey]
tmpobject[tmpitem] = `\$\{${tmpitem}\}`
}
//console.log("OBJ XML: ", tmpobject)
//newaction["body"] = XML.stringify(tmpobject, null, 2)
}
}
} else {
if (methodvalue["requestBody"]["content"]["example"] !== undefined) {
if (methodvalue["requestBody"]["content"]["example"]["example"] !== undefined) {
newaction["body"] = methodvalue["requestBody"]["content"]["example"]["example"]
//JSON.stringify(tmpobject, null, 2)
}
}
console.log(methodvalue["requestBody"]["content"])
if (methodvalue["requestBody"]["content"]["multipart/form-data"] !== undefined) {
if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"] !== null) {
if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") {
const fieldname = methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]["fieldname"]
if (fieldname !== undefined) {
console.log("FIELDNAME: ", fieldname)
newaction.file_field = fieldname["value"]
}
}
}
}
}
}
}
// HAHAHA wtf is this.
if (methodvalue.responses !== undefined && methodvalue.responses !== null) {
if (methodvalue.responses.default !== undefined) {
if (methodvalue.responses.default.content !== undefined) {
if (methodvalue.responses.default.content["text/plain"] !== undefined) {
if (methodvalue.responses.default.content["text/plain"]["schema"] !== undefined) {
if (methodvalue.responses.default.content["text/plain"]["schema"]["example"] !== undefined) {
newaction.example_response = methodvalue.responses.default.content["text/plain"]["schema"]["example"]
}
}
}
}
}
}
for (var key in methodvalue.parameters) {
const parameter = methodvalue.parameters[key]
const parameter = handleGetRef(methodvalue.parameters[key], data)
if (parameter.in === "query") {
var tmpaction = {
"description": parameter.description,
@@ -466,9 +627,12 @@ const AppCreator = (props) => {
}
} else if (parameter.in === "header") {
newaction.headers += `${parameter.name}=${parameter.example}\n`
} else {
console.log("WARNING: don't know how to handle this param: ", parameter)
}
}
if (newaction.name === "" || newaction.name === undefined) {
// Find a unique part of the string
// FIXME: Looks for length between /, find the one where they differ
@@ -582,6 +746,17 @@ const AppCreator = (props) => {
}
}
if (newActions.length > increaseAmount-1) {
setActionAmount(increaseAmount)
} else {
setActionAmount(newActions.length)
}
if (newActions.length > 1000) {
alert.error("Cut down actions from "+newActions.length+" to 999 because of limit")
newActions = newActions.slice(0,999)
}
setActions(newActions)
setIsAppLoaded(true)
}
@@ -659,6 +834,11 @@ const AppCreator = (props) => {
}
const regex = /[A-Za-z0-9 _]/g;
if (item.name === undefined) {
console.log("Skipping action ", item)
continue
}
const found = item.name.match(regex);
if (found !== null) {
item.name = found.join("")
@@ -668,17 +848,58 @@ const AppCreator = (props) => {
"responses": {
"default": {
"description": "default",
"schema": {}
"content": {
"text/plain": {
"schema": {
"type": "string",
"example": "",
},
},
},
}
},
"summary": item.name,
"operationId": item.name.split(" ").join("_"),
"description": item.description,
"parameters": []
"parameters": [],
"requestBody": {
"content": {
}
},
}
//console.log("ACTION: ", item)
if (item.example_response !== undefined && item.example_response.length > 0) {
// FIXME: Shallow copy of the string
var showResult = Object.assign("", item.example_response).trim()
showResult = showResult.split(" None").join(" \"None\"")
showResult = showResult.split("\'").join("\"")
showResult = showResult.split(" False").join(" false")
showResult = showResult.split(" True").join(" true")
var jsonvalid = true
try {
const tmp = String(JSON.parse(showResult))
if (!showResult.includes("{") && !showResult.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.type = "string"
if (jsonvalid) {
// FIXME: Add a JSON parser here - don't run it as a string.
data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = showResult
} else {
data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = item.example_response
}
}
if (item.queries.length > 0) {
for (var querykey in item.queries) {
const queryitem = item.queries[querykey]
@@ -773,16 +994,46 @@ const AppCreator = (props) => {
},
}
/*
data.paths[item.url][item.method.toLowerCase()]["requestBody"] = {
"description": "Generated by Shuffler.io",
"required": required,
"content": {
"example": {
"example": item.body,
},
},
}
*/
data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem)
}
// https://swagger.io/docs/specification/describing-request-body/file-upload/
if (item.file_field !== undefined && item.file_field !== null && item.file_field.length > 0) {
console.log("HANDLE FILEFIELD SAVE: ", item.file_field)
data.paths[item.url][item.method.toLowerCase()]["requestBody"]["content"]["multipart/form-data"] = {
"schema": {
"type": "object",
"properties": {
"fieldname": {
"type": "string",
"value": item.file_field,
},
},
},
}
console.log(data.paths[item.url][item.method.toLowerCase()]["requestBody"]["content"]["multipart/form-data"])
}
if (item.headers.length > 0) {
const required = false
const headersSplit = item.headers.split("\n")
for (var key in headersSplit) {
const header = headersSplit[key]
console.log("HEADER: ", header)
var key = ""
var value = ""
if (header.length > 0 && header.includes("= ")) {
@@ -989,6 +1240,7 @@ const AppCreator = (props) => {
"name": "",
"description": "",
"url": "",
"file_field": "",
"headers": "",
"paths": [],
"queries": [],
@@ -1102,14 +1354,14 @@ const AppCreator = (props) => {
null
:
<div>
{actions.map((data, index) => {
{actions.slice(0,actionAmount).map((data, index) => {
var error = data.errors.length > 0 ?
<Tooltip color="primary" title={data.errors.join("\n")} placement="bottom">
<ErrorOutline />
</Tooltip>
:
<Tooltip color="secondary" title={data.errors.join("\n")} placement="bottom">
<CheckCircleIcon />
<CheckCircleIcon style={{marginTop: 6}}/>
</Tooltip>
@@ -1127,6 +1379,7 @@ const AppCreator = (props) => {
}
const url = data.url
const hasFile = data["file_field"] !== undefined && data["file_field"] !== null && data["file_field"].length > 0
return (
<Paper style={actionListStyle}>
{error}
@@ -1137,15 +1390,23 @@ const AppCreator = (props) => {
setUrlPathQueries(data.queries)
setUrlPath(data.url)
setActionsModalOpen(true)
if (data["body"] !== undefined && data["body"] !== null && data["body"].length > 0) {
findBodyParams(data["body"])
}
if (hasFile) {
setFileUploadEnabled(true)
}
}}>
<div style={{display: "flex"}}>
<Chip
style={{backgroundColor: bgColor, color: "white", borderRadius: 5, minWidth: 80, marginRight: 10, marginTop: 2, cursor: "pointer", fontSize: 14,}}
label={data.method}
variant="contained"
/>
<span style={{fontSize: 16, marginTop: "auto", marginBottom: "auto",}}>
{url} - {data.name}
{hasFile ? <AttachFileIcon style={{height: 20, width: 20}} /> : null} {url} - {data.name}
</span>
</div>
</div>
@@ -1177,24 +1438,51 @@ const AppCreator = (props) => {
const setActionField = (field, value) => {
currentAction[field] = value
setCurrentAction(currentAction)
//setUrlPathQueries(currentAction.queries)
}
const findBodyParams = (body) => {
const regex = /\${(\w+)}/g
const found = body.match(regex)
if (found === null) {
setExtraBodyFields([])
} else {
setExtraBodyFields(found)
}
}
const bodyInfo = actionBodyRequest.includes(currentActionMethod) ?
<div>
Body - used as example in action argument
<div style={{marginTop: 10}}>
<b>Request Body</b>: {extraBodyFields.length > 0 ?
<Typography style={{display: "inline-block"}}>
Variables: {extraBodyFields.join(", ")}
</Typography>
:
<Typography style={{display: "inline-block"}}>
{`Add variables with \$\{ variable_name }`}
</Typography>
}
<TextField
required
style={{flex: "1", marginRight: "15px", backgroundColor: inputColor}}
fullWidth={true}
placeholder={'{\n\t"username": "testing@test.com"\n\t"name": "test testington"\n}'}
placeholder={'{\n\t"username": "${username}",\n\t"apikey": "${apikey}",\n\t"search": "1.2.3.5"}'}
margin="normal"
variant="outlined"
multiline
rows="5"
defaultValue={currentAction["body"]}
onChange={e => setActionField("body", e.target.value)}
onChange={e => {
setActionField("body", e.target.value)
findBodyParams(e.target.value)
}}
key={currentAction}
helperText={
<span style={{color:"white", marginBottom: "2px",}}>
Shows an example body to the user. ${} creates variables.
</span>
}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
@@ -1205,9 +1493,40 @@ const AppCreator = (props) => {
}}
/>
<div>
</div>
</div>
: null
const exampleResponse =
<div style={{}}>
<b>Example success response</b>
<TextField
required
style={{flex: "1", marginRight: "15px", backgroundColor: inputColor}}
fullWidth={true}
placeholder={'{\n\t"email": "testing@test.com",\n\t"firstname": "testing"\n}'}
margin="normal"
variant="outlined"
multiline
rows="2"
defaultValue={currentAction["example_response"]}
onChange={e => setActionField("example_response", e.target.value)}
helperText={<span style={{color:"white", marginBottom: "2px",}}>
Helps with autocompletion and understanding of the endpoint
</span>}
key={currentAction}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
},
style:{
color: "white",
},
}}
/>
</div>
const addActionToView = (errors) => {
currentAction.errors = errors
currentAction.queries = urlPathQueries
@@ -1329,7 +1648,7 @@ const AppCreator = (props) => {
const queries = values[1]
if (currentAction.paths !== paths && urlPath.length > 0) {
console.log("IN PATHS SETTER: !", paths)
//console.log("IN PATHS SETTER: !", paths)
setActionField("paths", paths)
}
@@ -1361,12 +1680,12 @@ const AppCreator = (props) => {
open={actionsModalOpen}
fullWidth
onClose={() => {
console.log("CLOSED?")
setUrlPath("")
setCurrentAction({
"name": "",
"description": "",
"url": "",
"file_field": "",
"headers": "",
"paths": [],
"queries": [],
@@ -1377,6 +1696,7 @@ const AppCreator = (props) => {
setCurrentActionMethod(apikeySelection[0])
setUrlPathQueries([])
setActionsModalOpen(false)
setFileUploadEnabled(false)
}}
>
<FormControl style={{backgroundColor: surfaceColor, color: "white",}}>
@@ -1453,11 +1773,13 @@ const AppCreator = (props) => {
id: 'method-option',
}}
>
{actionNonBodyRequest.map(data => (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data}>
{actionNonBodyRequest.map((data, index) => {
return (
<MenuItem key={index} style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data}
</MenuItem>
))}
)
})}
{actionBodyRequest.map(data => (
<MenuItem style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data}
@@ -1492,66 +1814,81 @@ const AppCreator = (props) => {
}}
onBlur={event => {
var parsedurl = event.target.value
if (parsedurl.startsWith("curl")) {
const request = parseCurl(event.target.value)
console.log(request)
if (request.method.toUpperCase() !== currentAction.Method) {
setCurrentActionMethod(request.method.toUpperCase())
setActionField("method", request.method.toUpperCase())
}
if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) {
const tmp = parsedurl.split(" ")
if (request.header !== undefined && request.header !== null) {
var headers = []
for (let [key, value] of Object.entries(request.header)) {
if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) {
continue
}
if (key === "Authorization" && authenticationOption === "Bearer auth") {
continue
}
headers += key+"="+value+"\n"
}
setActionField("headers", headers)
}
if (request.body !== undefined && request.body !== null) {
setActionField("body", request.body)
}
// Parse URL
if (request.url !== undefined) {
parsedurl = request.url
}
}
if (parsedurl !== undefined) {
if (parsedurl.includes("<") && parsedurl.includes(">")) {
parsedurl = parsedurl.split("<").join("{")
parsedurl = parsedurl.split(">").join("}")
}
if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) {
if (parsedurl !== undefined && parsedurl.includes(parameterName)) {
// Remove <> etc.
//
console.log("IT HAS THE PARAM NAME!")
const newurl = new URL(encodeURI(parsedurl))
newurl.searchParams.delete(parameterName)
parsedurl = decodeURI(newurl.href)
}
// Remove the base URL itself
if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) {
parsedurl = parsedurl.replace(baseUrl, "")
}
// Check URL query && headers
if (tmp.length > 1) {
parsedurl = tmp[1]
setActionField("url", parsedurl)
setUrlPath(parsedurl)
setCurrentActionMethod(tmp[0].toUpperCase())
setActionField("method", tmp[0].toUpperCase())
}
setUpdate(Math.random())
} else if (parsedurl.startsWith("curl")) {
const request = parseCurl(event.target.value)
if (request !== event.target.value) {
if (request.method.toUpperCase() !== currentAction.Method) {
setCurrentActionMethod(request.method.toUpperCase())
setActionField("method", request.method.toUpperCase())
}
if (request.header !== undefined && request.header !== null) {
var headers = []
for (let [key, value] of Object.entries(request.header)) {
if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) {
continue
}
if (key === "Authorization" && authenticationOption === "Bearer auth") {
continue
}
headers += key+"="+value+"\n"
}
setActionField("headers", headers)
}
if (request.body !== undefined && request.body !== null) {
setActionField("body", request.body)
}
// Parse URL
if (request.url !== undefined) {
parsedurl = request.url
}
}
console.log("PARSED: ", parsedurl)
if (parsedurl !== undefined) {
if (parsedurl.includes("<") && parsedurl.includes(">")) {
parsedurl = parsedurl.split("<").join("{")
parsedurl = parsedurl.split(">").join("}")
}
if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) {
if (parsedurl !== undefined && parsedurl.includes(parameterName)) {
// Remove <> etc.
//
console.log("IT HAS THE PARAM NAME!")
const newurl = new URL(encodeURI(parsedurl))
newurl.searchParams.delete(parameterName)
parsedurl = decodeURI(newurl.href)
}
// Remove the base URL itself
if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) {
parsedurl = parsedurl.replace(baseUrl, "")
}
// Check URL query && headers
setActionField("url", parsedurl)
setUrlPath(parsedurl)
}
}
}
@@ -1563,8 +1900,38 @@ const AppCreator = (props) => {
<Button color="primary" style={{marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} variant="outlined" onClick={() => {
addPathQuery()
}}>New query</Button>
{currentActionMethod === "POST" ?
<Button color="primary" variant={fileUploadEnabled ? "contained" : "outlined"} style={{marginLeft: 10, marginTop: "5px", marginBottom: "10px", borderRadius: "0px"}} onClick={() => {
setFileUploadEnabled(!fileUploadEnabled)
if (fileUploadEnabled && currentAction["file_field"].length > 0) {
setActionField("file_field", "")
}
setUpdate(Math.random())
}}>Enable Fileupload</Button>
: null}
{fileUploadEnabled ?
<TextField
required
style={{backgroundColor: inputColor, display: "inline-block",}}
placeholder={"file"}
margin="normal"
variant="outlined"
id="standard-required"
defaultValue={currentAction["file_field"]}
onChange={e => setActionField("file_field", e.target.value)}
helperText={<span style={{color:"white", marginBottom: "2px",}}>The File field to interact with</span>}
InputProps={{
classes: {
notchedOutline: classes.notchedOutline,
},
style:{
color: "white",
},
}}
/>
: null}
<div/>
Headers - static for the action
<b>Headers</b>: static for the action
<TextField
required
style={{flex: "1", marginRight: "15px", marginTop: "5px", backgroundColor: inputColor}}
@@ -1575,7 +1942,7 @@ const AppCreator = (props) => {
id="standard-required"
defaultValue={currentAction["headers"]}
multiline
rows="5"
rows="2"
onChange={e => setActionField("headers", e.target.value)}
helperText={<span style={{color:"white", marginBottom: "2px",}}>Headers that are part of the request. Default: EMPTY</span>}
InputProps={{
@@ -1588,6 +1955,8 @@ const AppCreator = (props) => {
}}
/>
{bodyInfo}
<Divider style={{backgroundColor: "rgba(255,255,255,0.5)", marginTop: 15, marginBottom: 15}} />
{exampleResponse}
</DialogContent>
<DialogActions>
<Button style={{borderRadius: "0px"}} onClick={() => {
@@ -1595,14 +1964,15 @@ const AppCreator = (props) => {
Cancel
</Button>
<Button color="primary" variant="outlined" style={{borderRadius: "0px"}} onClick={() => {
console.log(urlPathQueries)
console.log(urlPath)
// value={urlPath}
//console.log(urlPathQueries)
//console.log(urlPath)
//console.log(currentAction)
const errors = getActionErrors()
addActionToView(errors)
setActionsModalOpen(false)
setUrlPathQueries([])
setUrlPath("")
setFileUploadEnabled(false)
}}>
Submit
</Button>
@@ -1661,26 +2031,40 @@ const AppCreator = (props) => {
const actionView =
<div style={{color: "white"}}>
<h2>Actions</h2>
<h2>Actions ({actions.length})</h2>
Actions are the tasks performed by an app. Read more about actions and apps
<Link target="_blank" to="https://shuffler.io/docs/apps#actions" style={{textDecoration: "none", color: "#f85a3e"}}> here</Link>.
<div>
{loopActions}
<Button color="primary" style={{marginTop: "20px", borderRadius: "0px"}} variant="outlined" onClick={() => {
setCurrentAction({
"name": "",
"description": "",
"url": "",
"headers": "",
"queries": [],
"paths": [],
"body": "",
"errors": [],
"method": actionNonBodyRequest[0],
})
setCurrentActionMethod(actionNonBodyRequest[0])
setActionsModalOpen(true)
}}>New action</Button>
<div style={{display: "flex"}}>
<Button color="primary" style={{marginTop: "20px", borderRadius: "0px"}} variant="outlined" onClick={() => {
setCurrentAction({
"name": "",
"description": "",
"url": "",
"file_field": "",
"headers": "",
"queries": [],
"paths": [],
"body": "",
"errors": [],
"method": actionNonBodyRequest[0],
})
setCurrentActionMethod(actionNonBodyRequest[0])
setActionsModalOpen(true)
}}>New action</Button>
{actionAmount > 0 && actionAmount < actions.length ? null :
<Button color="primary" style={{marginTop: "20px", borderRadius: "0px", textAlign: "center"}} variant="outlined" onClick={() => {
if (actionAmount+increaseAmount > actions.length) {
setActionAmount(actions.length)
} else {
setActionAmount(actionAmount+increaseAmount)
}
}}>
See more actions
</Button>
}
</div>
</div>
</div>
+81 -30
View File
@@ -21,6 +21,7 @@ import {Link} from 'react-router-dom';
import Breadcrumbs from '@material-ui/core/Breadcrumbs';
import ReactJson from 'react-json-view'
import Chip from '@material-ui/core/Chip';
import { useTheme } from '@material-ui/core/styles';
import CachedIcon from '@material-ui/icons/Cached';
import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
@@ -46,6 +47,10 @@ const inputColor = "#383B40"
export const GetParsedPaths = (inputdata, basekey) => {
const splitkey = " > "
var parsedValues = []
if (inputdata === undefined || inputdata === null) {
return parsedValues
}
if (typeof(inputdata) !== "object") {
return parsedValues
}
@@ -106,9 +111,10 @@ export const GetParsedPaths = (inputdata, basekey) => {
const Apps = (props) => {
const { globalUrl, isLoggedIn, isLoaded } = props;
const { globalUrl, isLoggedIn, isLoaded, userdata } = props;
//const [workflows, setWorkflows] = React.useState([]);
const theme = useTheme();
const baseRepository = "https://github.com/frikky/shuffle-apps"
const alert = useAlert()
const [selectedApp, setSelectedApp] = React.useState({});
@@ -134,6 +140,7 @@ const Apps = (props) => {
const [field2, setField2] = React.useState("")
const [cursearch, setCursearch] = React.useState("")
const [sharingConfiguration, setSharingConfiguration] = React.useState("you")
const [downloadBranch, setDownloadBranch] = React.useState("master")
const [isDropzone, setIsDropzone] = React.useState(false);
const upload = React.useRef(null);
@@ -311,10 +318,18 @@ const Apps = (props) => {
boxColor = "orange"
}
if (data.invalid) {
boxColor = "red"
}
//<div style={{backgroundColor: theme.palette.inputColor, height: 100, width: 100, borderRadius: 3, verticalAlign: "middle", textAlign: "center", display: "table-cell"}}>
// <div style={{width: "100px", height: "100px", border: "1px solid black", verticalAlign: "middle", textAlign: "center", display: "table-cell"}}>
var imageline = data.large_image.length === 0 ?
<img alt={data.title} style={{width: 100, height: 100}} />
<img alt={data.title} style={{width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
:
<img alt={data.title} src={data.large_image} style={{width: 100, height: 100, maxWidth: "100%"}} />
<img alt={data.title} src={data.large_image} style={{maxWidth: 100, maxHeight: "100%", display: "block", margin: "0 auto"}} onLoad={(event) => {
//console.log("IMG LOADED!: ", event.target)
}} />
// FIXME - add label to apps, as this might be slow with A LOT of apps
var newAppname = data.name
@@ -336,7 +351,7 @@ const Apps = (props) => {
}
var description = data.description
const maxDescLen = 60
const maxDescLen = 51
if (description.length > maxDescLen) {
description = data.description.slice(0, maxDescLen)+"..."
}
@@ -359,8 +374,8 @@ const Apps = (props) => {
}
}
}}>
<Grid container style={{margin: 10, flex: "10"}}>
<ButtonBase>
<Grid container style={{margin: 10, flex: "10", maxHeight: 110, overflow: "hidden",}}>
<ButtonBase style={{backgroundColor: theme.palette.inputColor, border: 3}}>
{imageline}
</ButtonBase>
<div style={{marginLeft: "10px", marginTop: "5px", marginBottom: "5px", width: boxWidth, backgroundColor: boxColor}}>
@@ -515,9 +530,9 @@ const Apps = (props) => {
: null
var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ?
<img alt={selectedApp.title} style={{width: 100, height: 100}} />
<img alt={selectedApp.title} style={{width: 100, height: 100, backgroundColor: theme.palette.inputColor,}} />
:
<img alt={selectedApp.title} src={selectedApp.large_image} style={{width: 100, height: 100, maxWidth: "100%"}} />
<img alt={selectedApp.title} src={selectedApp.large_image} style={{maxHeight: 100, maxWidth: 100, backgroundColor: theme.palette.inputColor}} />
const GetAppExample = () => {
if (selectedAction.returns === undefined) {
@@ -588,10 +603,10 @@ const Apps = (props) => {
<div style={{marginRight: 15, marginTop: 10}}>
{imageline}
</div>
<div style={{maxWidth: "75%", overflow: "hidden"}}>
<div style={{maxWidth: "85%", overflow: "hidden"}}>
<h2 style={{marginTop: 20, marginBottom: 0, }}>{newAppname}</h2>
<p style={{marginTop: 5, marginBottom: 0,}}>Version {selectedApp.app_version}</p>
<p style={{marginTop: 5, marginBottom: 0}}>{description}</p>
<p style={{marginTop: 5, marginBottom: 0, maxHeight: 150, overflowY: "auto", overflowX: "hidden",}}>{description}</p>
</div>
</div>
{activateButton}
@@ -634,7 +649,7 @@ const Apps = (props) => {
updateAppField(selectedApp.id, "sharing", !selectedApp.sharing)
//setSelectedAction(event.target.value)
}}
style={{width: 150, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}}
style={{width: 150, backgroundColor: theme.palette.surfaceColor, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}}
SelectDisplayProps={{
style: {
marginLeft: 10,
@@ -699,7 +714,7 @@ const Apps = (props) => {
{selectedAction.parameters !== undefined && selectedAction.parameters !== null ?
<div style={{marginTop: 15, marginBottom: 15}}>
<b>Arguments</b>
<b>Parameters</b>
{selectedAction.parameters.map(data => {
var itemColor = "#f85a3e"
if (!data.required) {
@@ -802,12 +817,16 @@ const Apps = (props) => {
const reader = new FileReader();
reader.addEventListener('load', (e) => {
const content = e.target.result;
setOpenApiData(content);
setIsDropzone(isDropzone);
setOpenApiModal(true)
})
try {
reader.addEventListener('load', (e) => {
const content = e.target.result;
setOpenApiData(content);
setIsDropzone(isDropzone);
setOpenApiModal(true)
})
} catch (e) {
console.log("Error in dropzone: ", e)
}
reader.readAsText(files[0]);
};
@@ -907,7 +926,7 @@ const Apps = (props) => {
<div style={{marginTop: 15}}>
{apps.length > 0 ?
filteredApps.length > 0 ?
<div style={{height: "75vh", overflowY: "scroll"}}>
<div style={{height: "75vh", overflowY: "auto"}}>
{filteredApps.map(app => {
return (
appPaper(app)
@@ -959,6 +978,7 @@ const Apps = (props) => {
const parsedData = {
"url": url,
"branch": downloadBranch || 'master'
}
if (field1.length > 0) {
@@ -988,18 +1008,23 @@ const Apps = (props) => {
}
setIsLoading(false)
stop()
setValidation(false)
return response.json()
})
.then((responseJson) => {
console.log("DATA: ", responseJson)
if (responseJson.reason !== undefined) {
alert.error("Failed loading: "+responseJson.reason)
}
console.log("DATA: ", responseJson)
if (responseJson.reason !== undefined) {
alert.error("Failed loading: "+responseJson.reason)
}
})
.catch(error => {
console.log("ERROR: ", error.toString())
alert.error(error.toString())
stop()
setIsLoading(false)
setValidation(false)
})
}
@@ -1167,6 +1192,7 @@ const Apps = (props) => {
return
}
console.log("Validating response!")
validateOpenApi(responseJson)
})
.catch(error => {
@@ -1185,10 +1211,12 @@ const Apps = (props) => {
try {
return JSON.stringify(YAML.parse(apidata))
const parsed = YAML.parse(YAML.stringify(apidata))
//const parsed = YAML.parse(apidata))
return YAML.stringify(parsed)
} catch(error) {
console.log("YAML DECODE ERROR - TRY SOMETHING ELSE?: "+error)
setOpenApiError(error.toString())
setOpenApiError("Local error: "+ error.toString())
}
return ""
@@ -1197,19 +1225,23 @@ const Apps = (props) => {
// Sends the data to backend, which should return a version 3 of the same API
// If 200 - continue, otherwise, there's some issue somewhere
const validateOpenApi = (openApidata) => {
const newApidata = escapeApiData(openApidata)
var newApidata = escapeApiData(openApidata)
if (newApidata === "") {
// Used to return here
newApidata = openApidata
return
}
//console.log(newApidata)
setValidation(true)
fetch(globalUrl+"/api/v1/validate_openapi", {
method: 'POST',
method: 'POST',
headers: {
'Accept': 'application/json',
},
body: newApidata,
credentials: "include",
body: openApidata,
credentials: "include",
})
.then((response) => {
setValidation(false)
@@ -1302,7 +1334,7 @@ const Apps = (props) => {
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
defaultValue="https://github.com/frikky/shuffle-apps"
defaultValue={userdata.active_org.defaults.app_download_repo !== undefined && userdata.active_org.defaults.app_download_repo.length > 0 ? userdata.active_org.defaults.app_download_repo : "https://github.com/frikky/shuffle-apps"}
InputProps={{
style:{
color: "white",
@@ -1314,6 +1346,25 @@ const Apps = (props) => {
placeholder="https://github.com/frikky/shuffle-apps"
fullWidth
/>
<span style={{marginTop: 10}}>Branch (default value is "master"):</span>
<div style={{display: "flex"}}>
<TextField
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
defaultValue={userdata.active_org.defaults.app_download_branch !== undefined && userdata.active_org.defaults.app_download_branch.length > 0 ? userdata.active_org.defaults.app_download_branch : downloadBranch}
InputProps={{
style:{
color: "white",
height: "50px",
fontSize: "1em",
},
}}
onChange={e => setDownloadBranch(e.target.value)}
placeholder="master"
fullWidth
/>
</div>
<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
<div style={{display: "flex"}}>
+4 -3
View File
@@ -69,7 +69,7 @@ const LoginDialog = props => {
}),
)
.catch(error => {
setLoginInfo("Error in userdata: ", error)
setLoginInfo("Error logging in: ", error)
})
}
@@ -80,6 +80,7 @@ const LoginDialog = props => {
const onSubmit = (e) => {
e.preventDefault()
setLoginInfo("")
// FIXME - add some check here ROFL
// Just use this one?
@@ -114,7 +115,7 @@ const LoginDialog = props => {
}),
)
.catch(error => {
setLoginInfo("Error in userdata: " + error)
setLoginInfo("Error logging in: " + error)
});
} else {
url = baseurl + '/api/v1/users/register';
@@ -130,7 +131,7 @@ const LoginDialog = props => {
if (responseJson["success"] === false) {
setLoginInfo(responseJson["reason"])
} else {
setLoginInfo("Successful register :)")
setLoginInfo("Successful register!")
}
}),
)
+2 -2
View File
@@ -85,7 +85,7 @@ const Settings = (props) => {
const generateApikey = () => {
fetch(globalUrl+"/api/v1/generateapikey", {
method: 'GET',
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
@@ -99,7 +99,7 @@ const Settings = (props) => {
return response.json()
})
.then((responseJson) => {
.then((responseJson) => {
setUserSettings(responseJson)
})
.catch(error => {
+85 -64
View File
@@ -43,8 +43,40 @@ import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
const inputColor = "#383B40"
const surfaceColor = "#27292D"
export const validateJson = (showResult) => {
//showResult = showResult.split(" None").join(" \"None\"")
showResult = showResult.split(" False").join(" false")
showResult = showResult.split(" True").join(" true")
var jsonvalid = true
try {
const tmp = String(JSON.parse(showResult))
if (!showResult.includes("{") && !showResult.includes("[")) {
jsonvalid = false
}
} catch (e) {
showResult = showResult.split("\'").join("\"")
try {
const tmp = String(JSON.parse(showResult))
if (!showResult.includes("{") && !showResult.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
}
const result = jsonvalid ? JSON.parse(showResult) : showResult
//console.log("VALID: ", jsonvalid, result)
return {
"valid": jsonvalid,
"result": result,
}
}
const Workflows = (props) => {
const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies} = props;
const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies, userdata} = props;
document.title = "Shuffle - Workflows"
const alert = useAlert()
@@ -80,7 +112,7 @@ const Workflows = (props) => {
duration: 5000,
startImmediate: false,
callback: () => {
getWorkflowExecution(selectedWorkflow.id)
//getWorkflowExecution(selectedWorkflow.id)
}
})
@@ -183,7 +215,7 @@ const Workflows = (props) => {
if (responseJson.length > 0){
setSelectedWorkflow(responseJson[0])
getWorkflowExecution(responseJson[0].id)
//getWorkflowExecution(responseJson[0].id)
}
})
.catch(error => {
@@ -202,8 +234,8 @@ const Workflows = (props) => {
color: "#ffffff",
width: "100%",
display: "flex",
minWidth: 1366,
maxWidth: 1766,
minWidth: 1024,
maxWidth: 1024,
margin: "auto",
maxHeight: "90vh",
}
@@ -272,7 +304,7 @@ const Workflows = (props) => {
setSelectedExecution(responseJson[0])
setWorkflowExecutions(responseJson)
} else {
alert.info("Couldn't find executions for the workflow")
//alert.info("Couldn't find executions for the workflow")
setSelectedExecution({})
setWorkflowExecutions([])
}
@@ -298,7 +330,7 @@ const Workflows = (props) => {
if (response.status !== 200) {
console.log("Status not 200 for WORKFLOW EXECUTION :O!")
}
getWorkflowExecution(workflowid)
//getWorkflowExecution(workflowid)
return response.json()
})
@@ -361,10 +393,24 @@ const Workflows = (props) => {
data["owner"] = ""
for (var key in data.triggers) {
if (data.triggers[key].status == "running") {
data.triggers[key].status = "stopped"
const trigger = data.triggers[key]
if (trigger.app_name === "Shuffle Workflow") {
if (trigger.parameters.length > 2) {
trigger.parameters[2].value = ""
}
}
if (trigger.status == "running") {
trigger.status = "stopped"
}
}
for (var key in data.actions) {
data.actions[key].authentication_id = ""
}
//return
data["org"] = []
data["org_id"] = ""
data.execution_org = {"id": ""}
@@ -377,12 +423,15 @@ const Workflows = (props) => {
}
const copyWorkflow = (data) => {
data = JSON.parse(JSON.stringify(data))
alert.success("Copying workflow "+data.name)
console.log("data: ", data)
data.id = ""
data.name = data.name+"_copy"
//return
fetch(globalUrl+"/api/v1/workflows", {
method: 'POST',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
@@ -397,9 +446,9 @@ const Workflows = (props) => {
}
return response.json()
})
.then((responseJson) => {
.then((responseJson) => {
getAvailableWorkflows()
})
})
.catch(error => {
alert.error(error.toString())
});
@@ -480,7 +529,7 @@ const Workflows = (props) => {
<div style={{flex: "10",}} onClick={() => {
if (selectedWorkflow.id !== data.id) {
setSelectedWorkflow(data)
getWorkflowExecution(data.id)
//getWorkflowExecution(data.id)
}
}}>
<Typography variant="h6" style={{marginTop: 10, marginBottom: 0, }}>
@@ -537,7 +586,7 @@ const Workflows = (props) => {
<div style={{display: "flex", flex: 1}} onClick={() => {
if (selectedWorkflow.id !== data.id) {
setSelectedWorkflow(data)
getWorkflowExecution(data.id)
//getWorkflowExecution(data.id)
}
}}>
<Grid item style={{flex: "1", justifyContent: "center", overflow: "hidden", float: "bottom",}}>
@@ -684,22 +733,12 @@ const Workflows = (props) => {
}
var t = new Date(data.started_at*1000)
var jsonvalid = true
var showResult = data.result.trim()
showResult = replaceAll(showResult, " None", " \"None\"");
try {
const tmp = String(JSON.parse(showResult))
if (!tmp.includes("{") && !tmp.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
const validate = validateJson(showResult)
//console.log("VALID: ", jsonvalid)
if (jsonvalid) {
if (validate.valid) {
showResult = <ReactJson
src={JSON.parse(showResult)}
src={validate.result}
theme="solarized"
collapsed={collapseJson}
displayDataTypes={false}
@@ -707,7 +746,7 @@ const Workflows = (props) => {
/>
} else {
// FIXME - have everything parsed as json, either just for frontend
// or in the backend
// or in the backend?
/*
const newdata = {"result": data.result}
showResult = <ReactJson
@@ -780,23 +819,12 @@ const Workflows = (props) => {
var arg = null
if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) {
var jsonvalid = true
var showResult = selectedExecution.execution_argument.trim()
showResult = replaceAll(showResult, " None", " \"None\"");
const validate = validateJson(showResult)
try {
const tmp = String(JSON.parse(showResult))
if (!tmp.includes("{") && !tmp.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
arg = jsonvalid ?
arg = validate.valid ?
<ReactJson
src={JSON.parse(showResult)}
src={validate.result}
theme="solarized"
collapsed={true}
displayDataTypes={false}
@@ -807,22 +835,11 @@ const Workflows = (props) => {
var lastresult = null
if (selectedExecution.result !== undefined && selectedExecution.result.length > 0) {
var jsonvalid = true
var showResult = selectedExecution.result.trim()
showResult = replaceAll(showResult, " None", " \"None\"");
try {
const tmp = JSON.parse(showResult)
if (!tmp.includes("{") && !tmp.includes("[")) {
jsonvalid = false
}
} catch (e) {
jsonvalid = false
}
lastresult = jsonvalid ?
const validate = validateJson(showResult)
lastresult = validate.valid ?
<ReactJson
src={JSON.parse(showResult)}
src={validate.result}
theme="solarized"
collapsed={true}
displayDataTypes={false}
@@ -899,7 +916,7 @@ const Workflows = (props) => {
</div>
:
<h4>
There are no executions for this workflow yet
Executions have been moved to the Workflow itself. <div/><Link to={`/workflows/${selectedWorkflow.id}?view=executions`} style={{textDecoration: "none", color: "#f85a3e"}}>Click here to see them</Link>
</h4>
)
}
@@ -1204,7 +1221,7 @@ const Workflows = (props) => {
<div style={workflowViewStyle}>
<div style={{display: "flex"}}>
<div style={{flex: "4"}}>
<h2>Workflows</h2>
<h2>Workflows ({workflows.length})</h2>
</div>
<div style={{marginTop: 20}}>
{workflowButtons}
@@ -1222,24 +1239,27 @@ const Workflows = (props) => {
</div>
<div style={{flex: viewSize.executionsView, marginLeft: "10px", marginRight: "10px"}}>
<div style={{display: "flex"}}>
<div style={{flex: "10"}}>
<div style={{flex: 10}}>
<h2>Executions: {selectedWorkflow.name}</h2>
</div>
<div style={{flex: "1"}}>
{/*
<div style={{flex: 1}}>
<Button color="primary" style={{marginTop: "20px"}} variant="text" onClick={() => {
alert.info("Refreshing executions");
getWorkflowExecution(selectedWorkflow.id)
//getWorkflowExecution(selectedWorkflow.id)
}}>
<CachedIcon />
</Button>
</div>
*/}
</div>
<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
<div style={scrollStyle}>
<ExecutionsView />
</div>
</div>
<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "33%"}}>
{/*
<div style={{flex: viewSize.executionResults, marginLeft: "10px", marginRight: "10px", minWidth: "40%"}}>
<div style={{display: "flex"}}>
<div style={{flex: "1"}}>
<h2>Execution Timeline</h2>
@@ -1257,6 +1277,7 @@ const Workflows = (props) => {
<ExecutionDetails />
</div>
</div>
*/}
</div>
)
}
@@ -1348,7 +1369,7 @@ const Workflows = (props) => {
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
value={downloadUrl}
defaultValue={userdata.active_org.defaults.workflow_download_repo !== undefined && userdata.active_org.defaults.workflow_download_repo.length > 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl}
InputProps={{
style:{
color: "white",
@@ -1367,7 +1388,7 @@ const Workflows = (props) => {
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
value={downloadBranch}
defaultValue={userdata.active_org.defaults.workflow_download_branch !== undefined && userdata.active_org.defaults.workflow_download_branch.length > 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch}
InputProps={{
style:{
color: "white",
@@ -0,0 +1,9 @@
GOOS=linux go build main.go
zip function.zip main
aws lambda update-function-code \
--function-name shuffler-forwarder \
--runtime go1.* \
--zip-file fileb://function.zip \
--handler main \
--role arn:aws:iam::123456789012:role/execution_role
@@ -0,0 +1,35 @@
{
"name": "Shuffle",
"version": "1.0",
"author": "@frikkylikeme",
"url": "https://github.com/frikky/shuffle",
"license": "AGPL-V3",
"description": "Execute a workflow in Shuffle",
"dataTypeList": ["thehive:case", "thehive:alert"],
"command": "Shuffle/shuffle.py",
"baseConfig": "Shuffle",
"configurationItems": [
{
"name": "url",
"description": "The URL to your shuffle instance",
"type": "string",
"multi": false,
"required": true,
"defaultValue": "https://shuffler.io"
},
{
"name": "api_key",
"description": "The API key to your Shuffle user",
"type": "string",
"multi": false,
"required": true
},
{
"name": "workflow_id",
"description": "The ID of the workflow to execute",
"type": "string",
"multi": false,
"required": true
}
]
}
@@ -0,0 +1,28 @@
#!/usr/bin/env python
# encoding: utf-8
from cortexutils.responder import Responder
import requests
class Shuffle(Responder):
def __init__(self):
Responder.__init__(self)
self.api_key = self.get_param("config.api_key", "")
self.url = self.get_param("config.url", "")
self.workflow_id = self.get_param("config.workflow_id", "")
def run(self):
Responder.run(self)
parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id)
headers = {
"Authorization": "Bearer %s" % self.api_key
}
requests.post(parsed_url, headers=headers)
self.report({'message': 'message sent'})
if __name__ == '__main__':
Shuffle().run()
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
# Created by Shuffle, AS. <frikky@shuffler.io>.
WPYTHON_BIN="framework/python/bin/python3"
SCRIPT_PATH_NAME="$0"
DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)"
SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})"
case ${DIR_NAME} in
*/active-response/bin | */wodles*)
if [ -z "${WAZUH_PATH}" ]; then
WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)"
fi
PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py"
;;
*/bin)
if [ -z "${WAZUH_PATH}" ]; then
WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)"
fi
PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py"
;;
*/integrations)
if [ -z "${WAZUH_PATH}" ]; then
WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)"
fi
PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py"
;;
esac
${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@"
@@ -0,0 +1,177 @@
#!/usr/bin/env python
# Created by Shuffle, AS. <frikky@shuffler.io>.
# Based on the Slack integration using Webhooks
import json
import sys
import time
import os
try:
import requests
from requests.auth import HTTPBasicAuth
except Exception as e:
print("No module 'requests' found. Install: pip install requests")
sys.exit(1)
# ADD THIS TO ossec.conf configuration:
# <integration>
# <name>custom-shuffle</name>
# <hook_url>http://<IP>:3001/api/v1/hooks/<HOOK_ID></hook_url>
# <level>3</level>
# <alert_format>json</alert_format>
# </integration>
# Global vars
debug_enabled = False
pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
json_alert = {}
now = time.strftime("%a %b %d %H:%M:%S %Z %Y")
# Set paths
log_file = '{0}/logs/integrations.log'.format(pwd)
def main(args):
debug("# Starting")
# Read args
alert_file_location = args[1]
webhook = args[3]
debug("# Webhook")
debug(webhook)
debug("# File location")
debug(alert_file_location)
# Load alert. Parse JSON object.
with open(alert_file_location) as alert_file:
json_alert = json.load(alert_file)
debug("# Processing alert")
debug(json_alert)
debug("# Generating message")
msg = generate_msg(json_alert)
if isinstance(msg, str):
if len(msg) == 0:
return
debug(msg)
debug("# Sending message")
send_msg(msg, webhook)
def debug(msg):
if debug_enabled:
msg = "{0}: {1}\n".format(now, msg)
print(msg)
f = open(log_file, "a")
f.write(msg)
f.close()
# Skips container kills to stop self-recursion
def filter_msg(alert):
# These are things that recursively happen because Shuffle starts Docker containers
# Docker integration rules: https://github.com/wazuh/wazuh-ruleset/blob/ae36745db1d3f312db0392f5925c2f2b0ec009a9/rules/0560-docker_integration_rules.xml
skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928",]
if alert["rule"]["id"] in skip:
return False
#try:
# if "docker" in alert["rule"]["description"].lower() and "
#msg['text'] = alert.get('full_log')
#except:
# pass
#msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A"
return True
def generate_msg(alert):
if not filter_msg(alert):
print("Skipping rule %s" % alert["rule"]["id"])
return ""
level = alert['rule']['level']
if (level <= 4):
color = "good"
elif (level >= 5 and level <= 7):
color = "warning"
else:
color = "danger"
msg = {}
msg['color'] = color
msg['pretext'] = "WAZUH Alert"
msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A"
msg['text'] = alert.get('full_log')
msg['rule_id'] = alert["rule"]["id"]
msg['timestamp'] = alert["timestamp"]
msg['id'] = alert['id']
msg["all_fields"] = alert
#msg['fields'] = []
# msg['fields'].append({
# "title": "Agent",
# "value": "({0}) - {1}".format(
# alert['agent']['id'],
# alert['agent']['name']
# ),
# })
#if 'agentless' in alert:
# msg['fields'].append({
# "title": "Agentless Host",
# "value": alert['agentless']['host'],
# })
#msg['fields'].append({"title": "Location", "value": alert['location']})
#msg['fields'].append({
# "title": "Rule ID",
# "value": "{0} _(Level {1})_".format(alert['rule']['id'], level),
#})
#attach = {'attachments': [msg]}
return json.dumps(msg)
def send_msg(msg, url):
headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
res = requests.post(url, data=msg, headers=headers)
debug(res)
if __name__ == "__main__":
try:
# Read arguments
bad_arguments = False
if len(sys.argv) >= 4:
msg = '{0} {1} {2} {3} {4}'.format(
now,
sys.argv[1],
sys.argv[2],
sys.argv[3],
sys.argv[4] if len(sys.argv) > 4 else '',
)
debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug')
else:
msg = '{0} Wrong arguments'.format(now)
bad_arguments = True
# Logging the call
f = open(log_file, 'a')
f.write(msg + '\n')
f.close()
if bad_arguments:
debug("# Exiting: Bad arguments.")
sys.exit(1)
# Main function
main(sys.argv)
except Exception as e:
debug(str(e))
raise
+5
View File
@@ -0,0 +1,5 @@
<integration>
<name>custom-shuffle</name>
<hook_url>http://<IP>:3001/api/v1/hooks/webhook_<HOOK_ID></hook_url>
<alert_format>json</alert_format>
</integration>
+2 -2
View File
@@ -1,11 +1,11 @@
NAME=shuffle-orborus
VERSION=0.8.0
VERSION=0.8.54
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
#docker push frikky/$NAME:$VERSION
#docker push frikky/shuffle:$NAME
# docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
docker push frikky/shuffle:$NAME
docker push ghcr.io/frikky/$NAME:$VERSION
+186 -32
View File
@@ -21,17 +21,22 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
//"github.com/docker/docker/api/types/filters"
dockerclient "github.com/docker/docker/client"
"github.com/satori/go.uuid"
//network "github.com/docker/docker/api/types/network"
//natting "github.com/docker/go-connections/nat"
"github.com/mackerelio/go-osstat/cpu"
"github.com/mackerelio/go-osstat/memory"
)
// Starts jobs in bulk, so this could be increased
var sleepTime = 3
var maxConcurrency = 50
// Timeout if something rashes
var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT")
var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
var appSdkVersion = os.Getenv("SHUFFLE_APP_SDK_VERSION")
var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
@@ -47,6 +52,8 @@ var baseUrl = os.Getenv("BASE_URL")
var environment = os.Getenv("ENVIRONMENT_NAME")
var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE"))
var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
var workerIds = []string{}
type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
@@ -103,13 +110,25 @@ func getThisContainerId() {
out, err := exec.Command("bash", "-c", cmd).Output()
if err == nil {
containerId = strings.TrimSpace(string(out))
// cgroup error. Hardcoding this.
// https://github.com/moby/moby/issues/7015
//log.Printf("Checking if %s is in %s", ".scope", string(out))
if strings.Contains(string(out), ".scope") {
containerId = "shuffle-orborus"
//docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope
}
} else {
log.Printf("Failed getting container ID: %s", err)
containerId = "shuffle-orborus"
log.Printf("[WARNING] Failed getting container ID: %s", err)
}
}
log.Printf("Started with containerId %s", containerId)
}
// Deploys the internal worker whenever something happens
// https://docs.docker.com/engine/api/sdk/examples/
func deployWorker(image string, identifier string, env []string) {
// Binds is the actual "-v" volume.
hostConfig := &container.HostConfig{
@@ -124,23 +143,28 @@ func deployWorker(image string, identifier string, env []string) {
// form container id and use it as network source if it's not empty
if containerId != "" {
log.Printf("[INFO] Found container ID %s", containerId)
//log.Printf("[INFO] Found container ID %s", containerId)
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
} else {
//log.Printf("[INFO] Empty self container id, continue without NetworkMode")
}
if cleanupEnv == "true" {
hostConfig.AutoRemove = true
}
config := &container.Config{
Image: image,
Env: env,
}
log.Printf("Identifier: %s", identifier)
//log.Printf("[INFO] Identifier: %s", identifier)
cont, err := dockercli.ContainerCreate(
context.Background(),
config,
hostConfig,
nil,
nil,
identifier,
)
@@ -154,6 +178,7 @@ func deployWorker(image string, identifier string, env []string) {
config,
hostConfig,
nil,
nil,
identifier,
)
@@ -167,7 +192,8 @@ func deployWorker(image string, identifier string, env []string) {
}
}
err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
containerStartOptions := types.ContainerStartOptions{}
err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions)
if err != nil {
log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err)
return
@@ -195,6 +221,7 @@ func deployWorker(image string, identifier string, env []string) {
//}
} else {
log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment)
//workerIds = append(workerIds, cont.ID)
}
return
@@ -227,11 +254,11 @@ func initializeImages() {
ctx := context.Background()
if appSdkVersion == "" {
appSdkVersion = "0.8.0"
appSdkVersion = "0.8.5"
log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
}
if workerVersion == "" {
workerVersion = "0.8.0"
workerVersion = "0.8.54"
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
@@ -248,9 +275,7 @@ func initializeImages() {
// check whether they are the same first
images := []string{
//fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix),
//fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix),
fmt.Sprintf("frikky/shuffle:app_sdk"),
fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion),
// fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
@@ -275,6 +300,40 @@ func initializeImages() {
}
}
// Will be used for checking if there's enough to deploy based on a threshold
// E.g. having maximum CPU and maxmimum RAM
// Does this work containerized?
func getStats() {
fmt.Printf("\n")
memory, err := memory.Get()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
before, err := cpu.Get()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
time.Sleep(time.Duration(250) * time.Millisecond)
after, err := cpu.Get()
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
}
total := float64(after.Total - before.Total)
fmt.Printf("[INFO] memory total: %d bytes\n", memory.Total)
fmt.Printf("[INFO] memory used: %d bytes\n", memory.Used)
fmt.Printf("[INFO] cpu used : %f%%\n", float64(after.User-before.User)/total*100)
fmt.Printf("[INFO] cpu system: %f%%\n", float64(after.System-before.System)/total*100)
fmt.Printf("[INFO] cpu idle : %f%%\n", float64(after.Idle-before.Idle)/total*100)
fmt.Printf("\n")
}
// Initial loop etc
func main() {
log.Println("[INFO] Setting up execution environment")
@@ -302,7 +361,19 @@ func main() {
log.Printf("[INFO] Cleanup process running every %d seconds", workerTimeout)
}
go zombiecheck(workerTimeout)
if concurrencyEnv != "" {
//var concurrencyEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY")
tmpInt, err := strconv.Atoi(concurrencyEnv)
if err == nil {
maxConcurrency = tmpInt
log.Printf("[INFO] Max workflow execution concurrency set to %d", maxConcurrency)
} else {
log.Printf("[WARNING] Env SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY must be a number, not %s. Defaulted to %d", workerTimeoutEnv, maxConcurrency)
}
}
ctx := context.Background()
go zombiecheck(ctx, workerTimeout)
log.Printf("[INFO] Running towards %s with Org %s", baseUrl, orgId)
httpProxy := os.Getenv("HTTP_PROXY")
@@ -337,6 +408,8 @@ func main() {
},
}
//getStats()
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
@@ -367,13 +440,15 @@ func main() {
hasStarted := false
for {
//log.Printf("Prerequest")
//go getStats()
newresp, err := client.Do(req)
executionCount := getRunningWorkers(ctx, workerTimeout)
//log.Printf("Postrequest")
if err != nil {
log.Printf("[WARNING] Failed making request: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -394,7 +469,7 @@ func main() {
log.Printf("[ERROR] Failed reading body: %s", err)
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -408,7 +483,7 @@ func main() {
sleepTime = 10
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
@@ -423,13 +498,31 @@ func main() {
if len(executionRequests.Data) == 0 {
zombiecounter += 1
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(workerTimeout)
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
// Anything below here verifies concurrency virification
if executionCount >= maxConcurrency {
if zombiecounter*sleepTime > workerTimeout {
go zombiecheck(ctx, workerTimeout)
zombiecounter = 0
}
time.Sleep(time.Duration(sleepTime) * time.Second)
continue
}
//log.Printf("[INFO] Got %d new requests. Executing: %d. Max: %d", len(executionRequests.Data), executionCount, maxConcurrency)
allowed := maxConcurrency - executionCount
if len(executionRequests.Data) > allowed {
log.Printf("[WARNING] Throttle - Cutting down requests from %d to %d (MAX: %d, CUR: %d)", len(executionRequests.Data), allowed, maxConcurrency, executionCount)
executionRequests.Data = executionRequests.Data[0:allowed]
}
// New, abortable version. Should check executionid and remove everything else
var toBeRemoved ExecutionRequestWrapper
for _, execution := range executionRequests.Data {
@@ -453,6 +546,7 @@ func main() {
fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId),
fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
fmt.Sprintf("BASE_URL=%s", baseUrl),
fmt.Sprintf("CLEANUP=%s", cleanupEnv),
}
if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" {
@@ -466,7 +560,7 @@ func main() {
go deployWorker(workerImage, containerName, env)
log.Printf("[INFO] %s is deployed and to be removed from queue.", execution.ExecutionId)
log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1
toBeRemoved.Data = append(toBeRemoved.Data, execution)
}
@@ -528,25 +622,22 @@ func main() {
}
}
// FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk
func zombiecheck(workerTimeout int) error {
log.Println("[INFO] Looking for old containers")
ctx := context.Background()
// Is this ok to do with Docker? idk :)
func getRunningWorkers(ctx context.Context, workerTimeout int) int {
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
//Filters: filters.Args{
// map[string][]string{"ancestor": {"<imagename>:<version>"}},
//},
if err != nil {
log.Printf("[ERROR] Failed creating Containerlist: %s", err)
return err
log.Printf("[ERROR] Error getting containers: %s", err)
return maxConcurrency
}
containerNames := map[string]string{}
stopContainers := []string{}
removeContainers := []string{}
currenttime := time.Now().Unix()
counter := 0
for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) {
@@ -568,14 +659,75 @@ func zombiecheck(workerTimeout int) error {
for _, name := range container.Names {
// FIXME - add name_version_uid_uid regex check as well
if strings.HasPrefix(name, "/shuffle") {
if !strings.HasPrefix(name, "/worker") {
continue
}
log.Printf("[INFO] NAME: %s", name)
//log.Printf("Time: %d - %d", currenttime-container.Created, int64(workerTimeout))
if container.State == "running" && currenttime-container.Created < int64(workerTimeout) {
counter += 1
break
}
}
}
return counter
}
// FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk
func zombiecheck(ctx context.Context, workerTimeout int) error {
log.Println("[INFO] Looking for old containers (zombies)")
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
//log.Printf("Len: %d", len(containers))
if err != nil {
log.Printf("[ERROR] Failed creating Containerlist: %s", err)
return err
}
containerNames := map[string]string{}
stopContainers := []string{}
removeContainers := []string{}
log.Printf("[INFO] Baseimage: %s, Workertimeout: %d", baseimagename, int64(workerTimeout))
baseString := `/bin/sh -c 'python app.py --log-level DEBUG'`
for _, container := range containers {
// Skip random containers. Only handle things related to Shuffle.
if !strings.Contains(container.Image, baseimagename) && container.Command != baseString && container.Command != "./worker" {
shuffleFound := false
for _, item := range container.Labels {
if item == "shuffle" {
shuffleFound = true
break
}
}
// Check image name
if !shuffleFound {
log.Printf("Skipping: %s, %s", container.Labels, container.Image)
continue
}
//} else {
// log.Printf("NAME: %s", container.Image)
} else {
//log.Printf("Img: %s", container.Image)
//log.Printf("Names: %s", container.Names)
}
for _, name := range container.Names {
// FIXME - add name_version_uid_uid regex check as well
if strings.HasPrefix(name, "/shuffle") && !strings.HasPrefix(name, "/shuffle-subflow") {
continue
}
currenttime := time.Now().Unix()
//log.Printf("[INFO] (%s) NAME: %s. TIME: %d", container.State, name, currenttime-container.Created)
// Need to check time here too because a container can be removed the same instant as its created
currenttime := time.Now().Unix()
if container.State != "running" && currenttime-container.Created > int64(workerTimeout) {
removeContainers = append(removeContainers, container.ID)
containerNames[container.ID] = name
@@ -591,9 +743,10 @@ func zombiecheck(workerTimeout int) error {
}
// FIXME - add killing of apps with same execution ID too
log.Printf("[INFO] Should STOP %d containers.", len(stopContainers))
for _, containername := range stopContainers {
log.Printf("[INFO] Stopping and removing container %s", containerNames[containername])
go dockercli.ContainerStop(ctx, containername, nil)
dockercli.ContainerStop(ctx, containername, nil)
removeContainers = append(removeContainers, containername)
}
@@ -602,8 +755,9 @@ func zombiecheck(workerTimeout int) error {
Force: true,
}
log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers))
for _, containername := range removeContainers {
go dockercli.ContainerRemove(ctx, containername, removeOptions)
dockercli.ContainerRemove(ctx, containername, removeOptions)
}
return nil
+3 -1
View File
@@ -5,6 +5,8 @@ WORKDIR /app
RUN go get -u github.com/docker/docker/api/types
RUN go get -u github.com/docker/docker/api/types/container
RUN go get -u github.com/docker/docker/client
RUN go get -u github.com/gorilla/mux
RUN go get -u github.com/patrickmn/go-cache
COPY worker.go /app/worker.go
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
@@ -13,7 +15,7 @@ FROM alpine:3.12
ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.6.0
ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.8.5
RUN apk add --no-cache bash
COPY --from=builder /app/ /
+5 -3
View File
@@ -1,12 +1,14 @@
NAME=shuffle-worker
VERSION=0.8.0
VERSION=0.8.56
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
# Push both for now..
#docker push frikky/$NAME:$VERSION
docker push frikky/shuffle:$NAME
#docker push frikky/shuffle:$NAME_$VERSION
#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
#docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5
docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52
docker push ghcr.io/frikky/$NAME:$VERSION
File diff suppressed because it is too large Load Diff