#65: Added number(), parse() and more for data parsing
This commit is contained in:
frikky
+216
-18
@@ -102,6 +102,208 @@ class AppBase:
|
||||
|
||||
self.logger.info("AFTER FULLEXEC stream result")
|
||||
|
||||
# Gets the value at the paranthesis level you want
|
||||
def parse_nested_param(string, level):
|
||||
"""
|
||||
Generate strings contained in nested (), indexing i = level
|
||||
"""
|
||||
if len(re.findall("\(", string)) == len(re.findall("\)", string)):
|
||||
LeftRightIndex = [x for x in zip(
|
||||
[Left.start()+1 for Left in re.finditer('\(', string)],
|
||||
reversed([Right.start() for Right in re.finditer('\)', string)]))]
|
||||
|
||||
elif len(re.findall("\(", string)) > len(re.findall("\)", string)):
|
||||
return parse_nested_param(string + ')', level)
|
||||
elif len(re.findall("\(", string)) < len(re.findall("\)", string)):
|
||||
return parse_nested_param('(' + string, level)
|
||||
|
||||
else:
|
||||
return 'Failed to parse params'
|
||||
|
||||
try:
|
||||
return [string[LeftRightIndex[level][0]:LeftRightIndex[level][1]]]
|
||||
except IndexError:
|
||||
return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]]
|
||||
|
||||
# Finds the deepest level paranthesis in a string
|
||||
def maxDepth(S):
|
||||
current_max = 0
|
||||
max = 0
|
||||
n = len(S)
|
||||
|
||||
# Traverse the input string
|
||||
for i in range(n):
|
||||
if S[i] == '(':
|
||||
current_max += 1
|
||||
|
||||
if current_max > max:
|
||||
max = current_max
|
||||
elif S[i] == ')':
|
||||
if current_max > 0:
|
||||
current_max -= 1
|
||||
else:
|
||||
return -1
|
||||
|
||||
# finally check for unbalanced string
|
||||
if current_max != 0:
|
||||
return -1
|
||||
|
||||
return max-1
|
||||
|
||||
# Specific type parsing
|
||||
def parse_type(data, thistype):
|
||||
if data == None:
|
||||
return "Empty"
|
||||
|
||||
if "int" in thistype or "number" in thistype:
|
||||
try:
|
||||
return int(data)
|
||||
except ValueError:
|
||||
print("ValueError while casting %s" % data)
|
||||
return data
|
||||
if "lower" in thistype:
|
||||
return data.lower()
|
||||
if "upper" in thistype:
|
||||
return data.upper()
|
||||
if "trim" in thistype:
|
||||
return data.strip()
|
||||
if "strip" in thistype:
|
||||
return data.strip()
|
||||
if "split" in thistype:
|
||||
return data.split()
|
||||
if "len" in thistype or "length" in thistype:
|
||||
return len(data)
|
||||
if "parse" in thistype:
|
||||
splitvalues = []
|
||||
default_error = """Error. Expected syntax: parse(["hello","test1"],0:1)"""
|
||||
if "," in data:
|
||||
splitvalues = data.split(",")
|
||||
|
||||
for item in range(len(splitvalues)):
|
||||
splitvalues[item] = splitvalues[item].strip()
|
||||
else:
|
||||
return default_error
|
||||
|
||||
lastsplit = []
|
||||
if ":" in splitvalues[-1]:
|
||||
lastsplit = splitvalues[-1].split(":")
|
||||
else:
|
||||
try:
|
||||
lastsplit = [int(splitvalues[-1])]
|
||||
except ValueError:
|
||||
return default_error
|
||||
|
||||
try:
|
||||
parsedlist = ",".join(splitvalues[0:-1])
|
||||
if len(lastsplit) > 1:
|
||||
tmp = json.loads(parsedlist)[int(lastsplit[0]):int(lastsplit[1])]
|
||||
else:
|
||||
tmp = json.loads(parsedlist)[lastsplit[0]]
|
||||
|
||||
print(tmp)
|
||||
return tmp
|
||||
except IndexError as e:
|
||||
return default_error
|
||||
|
||||
# Parses the INNER value and recurses until everything is done
|
||||
def parse_wrapper(data):
|
||||
try:
|
||||
if "(" not in data or ")" not in data:
|
||||
return data
|
||||
except TypeError:
|
||||
return data
|
||||
|
||||
print("Running %s" % data)
|
||||
|
||||
# Look for the INNER wrapper first, then move out
|
||||
wrappers = ["int", "number", "lower", "upper", "trim", "strip", "split", "parse", "len", "length"]
|
||||
found = False
|
||||
for wrapper in wrappers:
|
||||
if wrapper not in data.lower():
|
||||
continue
|
||||
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
return data
|
||||
|
||||
# Do stuff here.
|
||||
innervalue = parse_nested_param(data, maxDepth(data)-0)
|
||||
outervalue = parse_nested_param(data, maxDepth(data)-1)
|
||||
print("INNER: ", outervalue)
|
||||
print("OUTER: ", outervalue)
|
||||
|
||||
if outervalue != innervalue:
|
||||
#print("Outer: ", outervalue, " inner: ", innervalue)
|
||||
for key in range(len(innervalue)):
|
||||
# Replace OUTERVALUE[key] with INNERVALUE[key] in data.
|
||||
print("Replace %s with %s in %s" % (outervalue[key], innervalue[key], data))
|
||||
data = data.replace(outervalue[key], innervalue[key])
|
||||
else:
|
||||
for thistype in wrappers:
|
||||
if thistype.lower() not in data.lower():
|
||||
continue
|
||||
|
||||
parsed_value = parse_type(innervalue[0], thistype.lower())
|
||||
return parsed_value
|
||||
|
||||
print("DATA: %s\n" % data)
|
||||
return parse_wrapper(data)
|
||||
|
||||
def parse_wrapper_start(data):
|
||||
newdata = []
|
||||
newstring = ""
|
||||
record = True
|
||||
paranCnt = 0
|
||||
for char in data:
|
||||
if char == "(":
|
||||
paranCnt += 1
|
||||
|
||||
if not record:
|
||||
record = True
|
||||
|
||||
if record:
|
||||
newstring += char
|
||||
|
||||
if paranCnt == 0 and char == " ":
|
||||
newdata.append(newstring)
|
||||
newstring = ""
|
||||
record = True
|
||||
|
||||
if char == ")":
|
||||
paranCnt -= 1
|
||||
|
||||
if paranCnt == 0:
|
||||
record = False
|
||||
|
||||
if len(newstring) > 0:
|
||||
newdata.append(newstring)
|
||||
|
||||
print(newdata)
|
||||
parsedlist = []
|
||||
non_string = False
|
||||
for item in newdata:
|
||||
ret = parse_wrapper(item)
|
||||
if not isinstance(ret, str):
|
||||
non_string = True
|
||||
|
||||
parsedlist.append(ret)
|
||||
|
||||
if len(parsedlist) > 0 and not non_string:
|
||||
return " ".join(parsedlist)
|
||||
elif len(parsedlist) == 1 and non_string:
|
||||
return parsedlist[0]
|
||||
else:
|
||||
print("Casting back to string because multi: ", parsedlist)
|
||||
newlist = []
|
||||
for item in parsedlist:
|
||||
try:
|
||||
newlist.append(str(item))
|
||||
except ValueError:
|
||||
newlist.append("parsing_error")
|
||||
return " ".join(newlist)
|
||||
|
||||
# Takes a workflow execution as argument
|
||||
# Returns a string if the result is single, or a list if it's a list
|
||||
def get_json_value(execution_data, input_data):
|
||||
@@ -185,7 +387,9 @@ class AppBase:
|
||||
|
||||
return basejson
|
||||
|
||||
|
||||
def parse_params(action, fullexecution, parameter):
|
||||
# Skip if it starts with $?
|
||||
jsonparsevalue = "$."
|
||||
match = ".*([$]{1}([a-zA-Z0-9()# _-]+\.?){1,})"
|
||||
|
||||
@@ -331,7 +535,8 @@ class AppBase:
|
||||
if check:
|
||||
return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
|
||||
|
||||
print(sourcevalue)
|
||||
|
||||
sourcevalue = parse_wrapper_start(sourcevalue)
|
||||
destinationvalue = condition["destination"]["value"]
|
||||
|
||||
if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE":
|
||||
@@ -341,6 +546,7 @@ class AppBase:
|
||||
if check:
|
||||
return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
|
||||
|
||||
destinationvalue = parse_wrapper_start(destinationvalue)
|
||||
available_checks = [
|
||||
"=",
|
||||
"equals",
|
||||
@@ -444,7 +650,6 @@ class AppBase:
|
||||
if check:
|
||||
raise "Value check error: %s" % Exception(check)
|
||||
|
||||
|
||||
# Custom format for ${name[0,1,2,...]}$
|
||||
submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})"
|
||||
actualitem = re.findall(submatch, value, re.MULTILINE)
|
||||
@@ -484,20 +689,10 @@ class AppBase:
|
||||
# With this parameter ready, add it to... a greater list of parameters. Rofl
|
||||
multi_parameters[parameter["name"]] = resultarray
|
||||
else:
|
||||
# Testing casting
|
||||
if value.lower() == "true":
|
||||
value = True
|
||||
elif value.lower() == "false":
|
||||
value = False
|
||||
else:
|
||||
constructors = [int]
|
||||
for c in constructors:
|
||||
try:
|
||||
value = c(value)
|
||||
print("Successfully parsed %s to %s", value, c)
|
||||
except ValueError:
|
||||
print("Failed to parse %s to %s", value, c)
|
||||
pass
|
||||
# Parses things like int(value)
|
||||
self.logger.info("Parsing wrapper data")
|
||||
value = parse_wrapper_start(value)
|
||||
self.logger.info("Parsing done: ", value)
|
||||
|
||||
params[parameter["name"]] = value
|
||||
multi_parameters[parameter["name"]] = value
|
||||
@@ -511,8 +706,11 @@ class AppBase:
|
||||
if isinstance(newres, str):
|
||||
result += newres
|
||||
else:
|
||||
try:
|
||||
result += str(result)
|
||||
except ValueError:
|
||||
result += "Failed autocasting. Can't handle %s type from function. Must be string" % type(newres)
|
||||
print("Can't handle type %s value from function" % (type(newres)))
|
||||
result += "Can't handle %s type from function" % type(newres)
|
||||
else:
|
||||
print("APP_SDK DONE: Starting multi execution with", multi_parameters)
|
||||
# 1. Use number of executions based on longest array
|
||||
@@ -579,7 +777,7 @@ class AppBase:
|
||||
print(f"Failed to execute: {e}")
|
||||
self.logger.exception(f"Failed to execute {e}-{action['id']}")
|
||||
action_result["status"] = "FAILURE"
|
||||
action_result["result"] = "Exception: %s" % e
|
||||
action_result["result"] = "General exception: %s" % e
|
||||
|
||||
action_result["completed_at"] = int(time.time())
|
||||
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
#!/bin/bash
|
||||
docker rmi frikky/shuffle:app_sdk --force
|
||||
docker build . -t frikky/shuffle:app_sdk
|
||||
docker push frikky/shuffle:app_sdk
|
||||
NAME=app_sdk
|
||||
VERSION=0.2.0
|
||||
|
||||
docker rmi frikky/shuffle:$NAME --force
|
||||
docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION
|
||||
|
||||
docker push frikky/shuffle:$NAME
|
||||
docker push frikky/$NAME:$VERSION
|
||||
|
||||
@@ -190,12 +190,14 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
|
||||
buildOptions,
|
||||
)
|
||||
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
|
||||
defer imageBuildResponse.Body.Close()
|
||||
_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
|
||||
if newerr != nil {
|
||||
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Read the STDOUT from the build process
|
||||
defer imageBuildResponse.Body.Close()
|
||||
_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
|
||||
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6130,7 +6130,8 @@ func init() {
|
||||
var err error
|
||||
ctx := context.Background()
|
||||
|
||||
log.Printf("Running INIT process")
|
||||
log.Printf("Starting Shuffle backend - initializing database connection")
|
||||
// option.WithoutAuthentication
|
||||
dbclient, err = datastore.NewClient(ctx, gceProject)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("DBclient error during init: %s", err))
|
||||
|
||||
+60
-28
@@ -89,6 +89,7 @@ type WorkflowApp struct {
|
||||
Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"`
|
||||
Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"`
|
||||
Owner string `json:"owner" datastore:"owner" yaml:"owner"`
|
||||
Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps
|
||||
PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"`
|
||||
Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
@@ -3853,22 +3854,9 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
|
||||
// Check the file
|
||||
filename := file.Name()
|
||||
if filename == "Dockerfile" {
|
||||
// Quick Dockerfile check
|
||||
//dockerfile := fmt.Sprintf("%sDockerfile", extra)
|
||||
//log.Printf("Handling Dockerfile %s", dockerfile)
|
||||
//dockerdata, err := ioutil.ReadFile(dockerfile)
|
||||
//if err != nil {
|
||||
// log.Printf("Failed to read dockerfile")
|
||||
// continue
|
||||
//}
|
||||
|
||||
//if len(dockerdata) == 0 {
|
||||
// log.Printf("Dockerfile is empty")
|
||||
// continue
|
||||
//}
|
||||
|
||||
log.Printf("Handle Dockerfile in location %s", extra)
|
||||
|
||||
// Set up to make md5 and check if the app is new (api.yaml+src/app.py+Dockerfile)
|
||||
// Check if Dockerfile, app.py or api.yaml has changed. Hash?
|
||||
//log.Printf("Handle Dockerfile in location %s", extra)
|
||||
// Try api.yaml and api.yml
|
||||
fullPath := fmt.Sprintf("%s%s", extra, "api.yaml")
|
||||
fileReader, err := fs.Open(fullPath)
|
||||
@@ -3881,21 +3869,55 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
|
||||
}
|
||||
}
|
||||
|
||||
readFile, err := ioutil.ReadAll(fileReader)
|
||||
appfileData, err := ioutil.ReadAll(fileReader)
|
||||
if err != nil {
|
||||
log.Printf("Failed reading %s: %s", fullPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(readFile) == 0 {
|
||||
if len(appfileData) == 0 {
|
||||
log.Printf("Failed reading %s - length is 0.", fullPath)
|
||||
continue
|
||||
}
|
||||
|
||||
var workflowapp WorkflowApp
|
||||
err = gyaml.Unmarshal(readFile, &workflowapp)
|
||||
// func md5sum(data []byte) string {
|
||||
// Make hash
|
||||
appPython := fmt.Sprintf("%s/src/app.py", extra)
|
||||
appPythonReader, err := fs.Open(appPython)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshaling %s: %s", fullPath, err)
|
||||
log.Printf("Failed to read %s", appPython)
|
||||
continue
|
||||
}
|
||||
|
||||
appPythonData, err := ioutil.ReadAll(appPythonReader)
|
||||
if err != nil {
|
||||
log.Printf("Failed reading %s: %s", appPython, err)
|
||||
continue
|
||||
}
|
||||
|
||||
dockerFp := fmt.Sprintf("%s/Dockerfile", extra)
|
||||
dockerfile, err := fs.Open(dockerFp)
|
||||
if err != nil {
|
||||
log.Printf("Failed to read %s", appPython)
|
||||
continue
|
||||
}
|
||||
|
||||
dockerfileData, err := ioutil.ReadAll(dockerfile)
|
||||
if err != nil {
|
||||
log.Printf("Failed to read dockerfile")
|
||||
continue
|
||||
}
|
||||
|
||||
combined := []byte{}
|
||||
combined = append(combined, appfileData...)
|
||||
combined = append(combined, appPythonData...)
|
||||
combined = append(combined, dockerfileData...)
|
||||
md5 := md5sum(combined)
|
||||
|
||||
var workflowapp WorkflowApp
|
||||
err = gyaml.Unmarshal(appfileData, &workflowapp)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshaling workflowapp %s: %s", fullPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -3915,15 +3937,27 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
|
||||
}
|
||||
|
||||
// Make an option to override existing apps?
|
||||
//Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps
|
||||
removeApps := []string{}
|
||||
skip := false
|
||||
for _, app := range allapps {
|
||||
if app.Name == workflowapp.Name && app.AppVersion == workflowapp.AppVersion {
|
||||
//log.Printf("App upload for %s:%s already exists.", app.Name, app.AppVersion)
|
||||
log.Printf("Overriding app %s:%s as it exists.", app.Name, app.AppVersion)
|
||||
// FIXME: Check if there's a new APP_SDK as well.
|
||||
// Skip this check if app_sdk is new.
|
||||
if app.Hash == md5 && app.Hash != "" {
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
|
||||
//log.Printf("Overriding app %s:%s as it exists but has different hash.", app.Name, app.AppVersion)
|
||||
removeApps = append(removeApps, app.ID)
|
||||
}
|
||||
}
|
||||
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
|
||||
err = checkWorkflowApp(workflowapp)
|
||||
if err != nil {
|
||||
log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion)
|
||||
@@ -3939,21 +3973,17 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
|
||||
}
|
||||
}
|
||||
|
||||
//if workflowapp.Environment == "" {
|
||||
// workflowapp.Environment = baseEnvironment
|
||||
//}
|
||||
|
||||
workflowapp.ID = uuid.NewV4().String()
|
||||
workflowapp.IsValid = true
|
||||
workflowapp.Verified = true
|
||||
workflowapp.Sharing = true
|
||||
workflowapp.Downloaded = true
|
||||
workflowapp.Hash = md5
|
||||
|
||||
err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
|
||||
if err != nil {
|
||||
log.Printf("Failed setting workflowapp: %s", err)
|
||||
continue
|
||||
//return err
|
||||
}
|
||||
|
||||
err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1)
|
||||
@@ -3975,6 +4005,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
|
||||
} else {
|
||||
if len(tags) > 0 {
|
||||
log.Printf("Successfully built image %s", tags[0])
|
||||
} else {
|
||||
log.Printf("Successfully built image docker img")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4493,10 +4493,16 @@ const AngularWorkflow = (props) => {
|
||||
executionData.results.map(data => {
|
||||
var showResult = data.result.trim()
|
||||
showResult.split(" None").join(" \"None\"")
|
||||
console.log("RESULT: ", showResult)
|
||||
|
||||
// showResult = replaceAll(showResult, " None", " \"None\"")
|
||||
// Super basic check.
|
||||
var jsonvalid = true
|
||||
try {
|
||||
JSON.parse(showResult)
|
||||
const tmp = String(JSON.parse(showResult))
|
||||
if (!tmp.includes("{") && !tmp.includes("[")) {
|
||||
jsonvalid = false
|
||||
}
|
||||
} catch (e) {
|
||||
jsonvalid = false
|
||||
}
|
||||
|
||||
@@ -518,7 +518,10 @@ const Workflows = (props) => {
|
||||
var showResult = data.result.trim()
|
||||
showResult = replaceAll(showResult, " None", " \"None\"");
|
||||
try {
|
||||
JSON.parse(showResult)
|
||||
const tmp = String(JSON.parse(showResult))
|
||||
if (!tmp.includes("{") && !tmp.includes("[")) {
|
||||
jsonvalid = false
|
||||
}
|
||||
} catch (e) {
|
||||
jsonvalid = false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user