marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added magic parser configuration to frontend with basic app sdk parsing

Browse Source
This commit is contained in:
frikky
2021-12-10 23:11:58 +01:00
parent 92a280bf2c
commit 5532296f74
16 changed files with 300 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/README.md
+10
View File
@@ -8,5 +8,15 @@ This is the SDK used for apps to behave like they should.
4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
5. Rebuild the Docker image (click load in GUI?)
## Cloud updates
1. Go to shuffle cloud on GCP
2. Go to Cloud Storage
3. Find shuffler.appspot.com
4. Navigate to generated_apps/baseline
5. Update SDK there. This will make all new apps run with the new SDK
## Cloud app force-updates
1. Run the "stitcher.go" program in the public shuffle-shared repository.
# LICENSE
Everything in here is MIT, not AGPLv3 as indicated by the license.
backend/app_sdk/app_base.py
+99 -3
View File
@@ -62,12 +62,100 @@ class AppBase:
if len(self.base_url) == 0:
self.base_url = self.url
# Checks output for whether it should be automatically parsed or not
def run_magic_parser(self, input_data):
if not isinstance(input_data, str):
return input_data
# Don't touch existing JSON/lists
if (input_data.startswith("[") and input_data.endswith("]")) or (input_data.startswith("{") and input_data.endswith("}")):
return input_data
# Don't touch large data.
if len(input_data) > 100000:
return input_data
new_input = input_data
try:
#new_input.strip()
new_input = input_data.split()
new_return = []
index = 0
for item in new_input:
splititem = ","
if ", " in item:
splititem = ", "
elif "," in item:
splititem = ","
else:
new_return.append(item)
index += 1
continue
#print("FIX ITEM %s" % item)
for subitem in item.split(splititem):
new_return.insert(index, subitem)
index += 1
# Prevent large data or infinite loops
if index > 10000:
return input_data
fixed_return = []
for item in new_return:
if not item:
continue
if not isinstance(item, str):
fixed_return.append(item)
continue
if item.endswith(","):
item = item[0:-1]
fixed_return.append(item)
new_input = fixed_return
except Exception as e:
self.logger.info(f"[DEBUG] Failed to run magic parser (2): {e}")
return input_data
try:
new_input = input_data.split()
except Exception as e:
self.logger.info(f"[DEBUG] Failed to run magic parser (1): {e}")
return input_data
# Won't ever touch this one?
if isinstance(input_data, list) or isinstance(input_data, object):
try:
return json.dumps(new_input)
except Exception as e:
self.logger.info(f"[DEBUG] Failed to run magic parser: {e}")
return new_input
# FIXME: Add more info like logs in here.
# Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2
def send_result(self, action_result, headers, stream_path):
if action_result["status"] == "EXECUTING":
action_result["status"] = "FAILURE"
try:
if self.original_action["run_magic_output"] == True:
self.logger.warning("[INFO] Action result ran with Magic parser output.")
action_result["result"] = self.run_magic_parser(action_result["result"])
else:
self.logger.warning("[ERROR] Magic output not defined.")
except Exception as e:
self.logger.warning("[ERROR] Failed to run magic autoparser: {e}")
pass
# Try it with some magic
self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""")
# FIXME: Add cleanup of parameters to not send to frontend here
@@ -981,12 +1069,20 @@ class AppBase:
except json.decoder.JSONDecodeError:
pass
self.action_result["result"] = "Bad result from backend: %d" % ret.status_code
self.action_result["result"] = json.dumps({
"success": False,
"reason": f"Bad result from backend during startup of app: {ret.status_code}",
"extended_reason": f"{ret.text}"
})
self.send_result(self.action_result, headers, stream_path)
return
except requests.exceptions.ConnectionError as e:
self.logger.info("[DEBUG] FullExec Connectionerror: %s" % e)
self.action_result["result"] = "Connection error during startup: %s" % e
self.action_result["result"] = json.dumps({
"success": False,
"reason": f"Connection error during startup: {e}"
})
self.send_result(self.action_result, headers, stream_path)
return
else:
@@ -2805,7 +2901,7 @@ class AppBase:
# Dump the result as a string of a list
#self.logger.info("RESULTS: %s" % results)
if isinstance(results, list) or isinstance(results, dict):
self.logger.info("JSON OBJECT? ", json_object)
self.logger.info(f"JSON OBJECT? {json_object}")
# This part is weird lol
if json_object:
backend/app_sdk/build.sh
+1 -1
View File
@@ -3,7 +3,7 @@
### DEFAULT
NAME=shuffle-app_sdk
VERSION=0.9.35
VERSION=0.9.40
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly
backend/app_sdk/requirements.txt
+1 -1
View File
@@ -1,7 +1,7 @@
urllib3==1.26.5
requests==2.25.1
MarkupSafe==2.0.1
liquidpy==0.7.2
liquidpy==0.7.3
flask[async]==2.0.2
#waitress==2.0.0
#flask==1.1.2
backend/go-app/docker.go
+1
View File
@@ -299,6 +299,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
}
if !downloaded {
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
}
//baseDockerName
backend/go-app/go.mod
+1 -1
View File
@@ -2,7 +2,7 @@ module main
go 1.15
//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
//replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch
backend/go-app/walkoff.go
+21 -2
View File
@@ -2467,6 +2467,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId
log.Printf("[INFO] Should send email to %s during execution.", email)
}
if strings.Contains(triggerType, "sms") {
action := shuffle.CloudSyncJob{
Type: "user_input",
@@ -2491,7 +2492,11 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId
return err
}
log.Printf("Should send SMS to %s during execution.", sms)
log.Printf("[DEBUG] Should send SMS to %s during execution.", sms)
}
if strings.Contains(triggerType, "subflow") {
log.Printf("[DEBUG] Should run a subflow with the result for user input.")
}
return nil
@@ -2927,7 +2932,21 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("Failed image build memory: %s", err)
orgId := ""
log.Printf("[DEBUG] Failed image build memory. Creating notification with org %#v: %s", orgId, err)
if len(item.Tags) > 0 {
err = shuffle.CreateOrgNotification(
ctx,
fmt.Sprintf("App failed to build"),
fmt.Sprintf("The app %s with image %s failed to build. Check backend logs with docker! docker logs shuffle-backend", item.Tags[0], item.Extra),
fmt.Sprintf("/apps"),
orgId,
false,
)
}
} else {
if len(item.Tags) > 0 {
log.Printf("[INFO] Successfully built image %s", item.Tags[0])