marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed issues with semantic versioning for apps

Browse Source
This commit is contained in:
frikky
2021-06-17 09:05:13 +02:00
parent 642b589953
commit d2b50cfa14
10 changed files with 133 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/install-guide.md
+2
View File
@@ -1,6 +1,8 @@
# Installation guide
Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the [updating guide](https://shuffler.io/docs/configuration#updating_shuffle)
This document outlines a an introduction environment which is not scalable. [Read here](https://shuffler.io/docs/configuration#production_readiness) for information on production readiness.
# Docker - *nix
The Docker setup is done with docker-compose
backend/app_sdk/app_base.py
+2
View File
@@ -514,6 +514,7 @@ class AppBase:
# return
print("[INFO] Multiplier length: %d" % len(param_multiplier))
tmp = ""
for subparams in param_multiplier:
print(f"SUBPARAMS IN MULTI: {subparams}")
try:
@@ -2306,6 +2307,7 @@ class AppBase:
#newres = await func(**params)
#print("PARAMS: %s" % params)
newres = ""
while True:
try:
newres = await func(**params)
backend/go-app/docker.go
+10 -4
View File
@@ -697,7 +697,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
// REBUILDS THE APP
if len(img.ID) == 0 {
if len(img2.ID) == 0 {
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 0)
log.Printf("[INFO] Getting workflowapps for a rebuild. Got %d with err %#v", len(workflowapps), err)
if err == nil {
imageName := ""
@@ -720,16 +720,22 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
}
if len(imageName) > 0 && len(imageVersion) > 0 {
log.Printf("Looking for appname %s with version %s", imageName, imageVersion)
foundApp := shuffle.WorkflowApp{}
imageName = strings.ToLower(imageName)
imageVersion = strings.ToLower(imageVersion)
log.Printf("[DEBUG] Looking for appname %s with version %s", imageName, imageVersion)
for _, app := range workflowapps {
if strings.ToLower(strings.Replace(app.Name, " ", "_", -1)) == imageName && app.AppVersion == imageVersion {
if app.Generated {
log.Printf("[DEBUG] Found matching app %s:%s - %s", imageName, imageVersion, app.ID)
foundApp = app
break
} else {
log.Printf("[WARNING] Trying to rebuild app that isn't generated - not allowed. Looking further.")
}
break
//break
}
}
@@ -767,7 +773,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
}
//log.Printf("[INFO] Img found (%s): %#v", tagFound, img)
log.Printf("[INFO] Img found to be downloaded: %s", tagFound)
log.Printf("[INFO] Img found to be downloaded by client: %s", tagFound)
newClient, err := newdockerclient.NewClientFromEnv()
if err != nil {
backend/go-app/go.mod
+1 -1
View File
@@ -23,7 +23,7 @@ require (
github.com/elastic/go-elasticsearch v0.0.0 // indirect
github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect
github.com/frikky/kin-openapi v0.39.0
github.com/frikky/shuffle-shared v0.0.63
github.com/frikky/shuffle-shared v0.0.66
github.com/fsouza/go-dockerclient v1.7.2 // indirect
github.com/ghodss/yaml v1.0.0
github.com/go-git/go-billy/v5 v5.0.0
backend/go-app/go.sum
+2
View File
@@ -179,6 +179,8 @@ github.com/frikky/shuffle-shared v0.0.62 h1:1M8y7rX8nQW7072+bUD4vgHcf65AG0kJ8m3i
github.com/frikky/shuffle-shared v0.0.62/go.mod h1:BknTfpun3qte5bumR3OqQHf9XWPIsyj8woiXCjIlbBc=
github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzkEcUvu+c=
github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
github.com/frikky/shuffle-shared v0.0.65 h1:TvISX6WE1Y7M2UJ3YuMv0PtgRPWtMj5iGiArVmv6SI8=
github.com/frikky/shuffle-shared v0.0.65/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo=
github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8=
github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=
backend/go-app/main.go
+18 -10
View File
@@ -3091,7 +3091,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
var test Test
err := json.Unmarshal(body, &test)
if err != nil {
log.Printf("Failed unmarshalling test: %s", err)
log.Printf("[WARNING] Failed unmarshalling test: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -3107,7 +3107,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
ctx := context.Background()
app, err := shuffle.GetApp(ctx, test.Id, user)
if err != nil {
log.Printf("Error getting app when editing: %s", app.Name)
log.Printf("[WARNING] Error getting app when editing: %s", app.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -3325,15 +3325,19 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
// FIXME: Might cause versioning issues if we re-use the same!!
// FIXME: Need a way to track different versions of the same app properly.
// Hint: Save API.id somewhere, and use newmd5 to save latest version
err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed)
if err != nil {
log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err)))
if len(user.Id) > 0 {
err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed)
if err != nil {
log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err)))
}
shuffle.SetOpenApiDatastore(ctx, api.ID, parsed)
}
// Backup every single one
shuffle.SetOpenApiDatastore(ctx, api.ID, parsed)
/*
err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id)
@@ -3351,6 +3355,8 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) {
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
if len(user.Id) > 0 {
@@ -3475,6 +3481,8 @@ func handleAppHotload(ctx context.Context, location string, forceUpdate bool) er
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
//shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
return nil
@@ -4073,7 +4081,7 @@ func runInitEs(ctx context.Context) {
}
// Getting apps to see if we should initialize a test
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
if err != nil && len(workflowapps) == 0 {
log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
@@ -4713,7 +4721,7 @@ func runInit(ctx context.Context) {
}
// Getting apps to see if we should initialize a test
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %s", len(workflowapps), err)
if err != nil && len(workflowapps) == 0 {
log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
backend/go-app/walkoff.go
+17 -5
View File
@@ -2666,7 +2666,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
//return
}
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
if err != nil {
log.Printf("Failed getting apps (getworkflowapps): %s", err)
resp.WriteHeader(401)
@@ -2787,7 +2787,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) {
// FIXME - continue the search here with github repos etc.
// Caching might be smart :D
ctx := context.Background()
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
if err != nil {
log.Printf("Error: Failed getting workflowapps: %s", err)
resp.WriteHeader(401)
@@ -3005,7 +3005,9 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
}
ctx := context.Background()
cacheKey := fmt.Sprintf("workflowapps-sorted-500")
cacheKey := fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-0")
shuffle.DeleteCache(ctx, cacheKey)
@@ -3046,6 +3048,8 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -3178,6 +3182,8 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -3186,7 +3192,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
ctx := context.Background()
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
appCounter := 0
if err != nil {
log.Printf("Failed to get existing generated apps")
@@ -3317,6 +3323,8 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
}
} else {
//log.Printf("Skipped upload of %s (%s)", api.Name, api.ID)
@@ -3741,6 +3749,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
if len(extra) == 0 {
@@ -3811,7 +3821,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
}
ctx := context.Background()
allapps, err := shuffle.GetAllWorkflowApps(ctx, 500)
allapps, err := shuffle.GetAllWorkflowApps(ctx, 1000)
if err != nil {
log.Printf("Failed getting apps to verify: %s", err)
resp.WriteHeader(401)
@@ -3914,6 +3924,8 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
frontend/src/views/AngularWorkflow.jsx
+9
View File
@@ -2742,6 +2742,15 @@ const AngularWorkflow = (props) => {
for (var key in filteredelements) {
}
var copyText = document.getElementById("copy_element_shuffle")
if (copyText !== undefined && copyText !== null) {
const clipboard = navigator.clipboard
if (clipboard === undefined) {
alert.error("Can only copy over HTTPS (port 3443)")
return
}
}
console.log("FILTERED: ", filteredelements)
}
break;
frontend/src/views/Apps.jsx
+1 -1
View File
@@ -665,7 +665,7 @@ const Apps = (props) => {
</Typography>
</div>
</div>
{selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 0 ?
{selectedApp.versions !== null && selectedApp.versions !== undefined && selectedApp.versions.length > 1 ?
<Select
defaultValue={selectedApp.app_version}
onChange={(event) => {
functions/onprem/worker/worker.go
+71 -42
View File
@@ -310,7 +310,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
ShowStdout: true,
}
exit := true
exit := false
out, err := cli.ContainerLogs(ctx, cont.ID, logOptions)
if err != nil {
log.Printf("[INFO] Failed getting logs: %s", err)
@@ -318,11 +318,13 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
buf := new(strings.Builder)
io.Copy(buf, out)
logs := buf.String()
// FIXME: Re-add log tracking which can be sent to backend
//allLogs[actionId] = logs
if stats.ContainerJSONBase.State.Status == "exited" && strings.Contains(logs, "Normal execution.") {
if stats.ContainerJSONBase.State.Status == "exited" && !strings.Contains(logs, "Normal execution.") {
log.Printf("[WARNING] BAD Execution Logs for %s: %s", actionId, logs)
exit = false
exit = true
}
}
@@ -994,20 +996,33 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
image = images[2]
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (3)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
log.Printf("[WARNING] Failed CLEANUP execution. Downloading image %s remotely.", image)
err := downloadDockerImageBackend(topClient, image)
if err == nil {
log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
err := downloadDockerImageBackend(topClient, image)
executed := false
if err == nil {
log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
//err = deployApp(dockercli, image, identifier, env, workflow, action.ID)
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (41)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
} else {
executed = true
}
}
if !executed {
image = images[2]
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (3)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
//log.Printf("[WARNING] Failed CLEANUP execution. Downloading image %s remotely.", image)
log.Printf("[WARNING] Failed to download image %s (CLEANUP): %s", image, err)
reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
@@ -1032,22 +1047,22 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
log.Printf("[INFO] Successfully downloaded %s", image)
}
}
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist")
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (7)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist")
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (7)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
if strings.Contains(err.Error(), "No such image") {
//log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err)
log.Printf("[ERROR] Image doesn't exist. Shutting down")
log.Printf("[DEBUG] Shutting down (8)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
if strings.Contains(err.Error(), "No such image") {
//log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err)
log.Printf("[ERROR] Image doesn't exist. Shutting down")
log.Printf("[DEBUG] Shutting down (8)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
}
}
}
@@ -1071,20 +1086,34 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
image = images[2]
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (11)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub.", image)
err := downloadDockerImageBackend(topClient, image)
if err == nil {
log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
log.Printf("[DEBUG] Failed deploy. Downloading image %s", image)
err := downloadDockerImageBackend(topClient, image)
executed := false
if err == nil {
log.Printf("[DEBUG] Downloaded image %s from backend (CLEANUP)", image)
//err = deployApp(dockercli, image, identifier, env, workflow, action.ID)
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (40)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
} else {
executed = true
}
}
if !executed {
image = images[2]
err = deployApp(dockercli, image, identifier, env, workflowExecution, action.ID)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
if strings.Contains(err.Error(), "exited prematurely") {
log.Printf("[DEBUG] Shutting down (11)")
shutdown(workflowExecution, action.ID, fmt.Sprintf("%s", err.Error()), true)
}
log.Printf("[WARNING] Failed deploying image THREE TIMES. Attempting to download %s as last resort from backend and dockerhub.", image)
reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") {
log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image)