marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '2.0.0'

Browse Source
This commit is contained in:
Aditya
2024-12-05 05:20:50 +05:30
parent 96bcd652ef dcc513ea8d
commit 5fcf114b5f
136 changed files with 45316 additions and 19511 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/Dockerfile
-21
View File
@@ -1,21 +0,0 @@
#FROM python:3.9.1-alpine as base
FROM python:3.10.0-alpine as base
#FROM python:3.11.3-alpine as base
FROM base as builder
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils
RUN mkdir /install
WORKDIR /install
FROM base
#--no-cache
RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils
COPY --from=builder /install /usr/local
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/Dockerfile_alpine_grpc
-42
View File
@@ -1,42 +0,0 @@
FROM python:3.10.0-alpine as base
FROM base as builder
RUN apk --no-cache add --update \
alpine-sdk \
build-base \
g++ \
gcc \
libffi \
libffi-dev \
libstdc++ \
linux-headers \
musl-dev \
openssl-dev \
tzdata \
coreutils
RUN pip install --upgrade pip && \
pip install --prefix="/install" --no-cache-dir grpcio grpcio-tools && \
apk del --purge \
g++ \
gcc \
musl-dev \
libffi-dev \
libstdc++ \
build-base \
linux-headers
RUN mkdir -p /install
WORKDIR /install
FROM base
#--no-cache
RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils
COPY --from=builder /install /usr/local
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/Dockerfile_blackarch
-19
View File
@@ -1,19 +0,0 @@
FROM blackarchlinux/blackarch as base
FROM base as builder
RUN /bin/pacman -Syu --noconfirm
RUN /bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
FROM base
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/Dockerfile_kali
-19
View File
@@ -1,19 +0,0 @@
FROM kalilinux/kali-rolling as base
FROM base as builder
RUN apt-get update
RUN apt-get dist-upgrade -y
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
FROM base
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/Dockerfile_ubuntu
-22
View File
@@ -1,22 +0,0 @@
FROM ubuntu as base
FROM base as builder
RUN apt-get update
RUN apt-get dist-upgrade -y
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
FROM base
RUN apt-get update
RUN apt-get dist-upgrade -y
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
backend/app_sdk/LICENSE
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2020 Frikkylikeme
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
backend/app_sdk/README.md
Executable → Regular
+2 -22
View File
@@ -1,22 +1,2 @@
# app_sdk.py
This is the SDK used for apps to behave like they should.
## If you want to update apps.. PS: downloads from docker hub do overrides.. :)
1. Write your code & check if runtime works
2. Build app_base image
3. docker rm $(docker ps -aq) # Remove all stopped containers
4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
5. Rebuild the Docker image (click load in GUI?)
## Cloud updates
1. Go to shuffle cloud on GCP
2. Go to Cloud Storage
3. Find shuffler.appspot.com
4. Navigate to generated_apps/baseline
5. Update SDK there. This will make all new apps run with the new SDK
## Cloud app force-updates
1. Run the "stitcher.go" program in the public shuffle-shared repository.
# LICENSE
Everything in here is MIT, not AGPLv3 as indicated by the license.
## CHANGES
In November 2024, we moved this to its own repistory: https://github.com/shuffle/app_sdk
backend/app_sdk/__init__.py
View File
backend/app_sdk/app_base.py
-4143
View File
File diff suppressed because it is too large Load Diff
backend/app_sdk/autocorrect_test.py
-187
View File
@@ -1,187 +0,0 @@
import re
import json
input_data = """{
"test4": $test,
"test5": ,
"test6": "what"
}
"""
input_data = """{
"test0": {{ '' | default: [] }},
"test": {{ | default: [] }},
"test2": {{ $test.asd | default: [] }},
"test3": {{ {"key": "val} | default: [] }},
"test4": $test,
"test5": ,
"test6": "what"
}
"""
liquiddata = "{{ $test.asd | some other stuff {{ $test.xyz | more stuff"
pattern = r'\{\{\s*\$[^|}]+\s*\|'
replaced_data = re.sub(pattern, "{{ '' |", liquiddata)
print(replaced_data)
def patternfix_string(liquiddata, patterns, regex_patterns, inputtype="liquid"):
if not inputtype or inputtype == "liquid":
if "{{" not in liquiddata or "}}" not in liquiddata:
return liquiddata
elif inputtype == "json":
liquiddata = liquiddata.strip()
# Validating if it looks like json or not
if liquiddata[0] == "{" and liquiddata[len(liquiddata)-1] == "}":
pass
else:
if liquiddata[0] == "[" and liquiddata[len(liquiddata)-1] == "]":
pass
else:
return liquiddata
# If it's already json, don't touch it
try:
json.loads(liquiddata)
return liquiddata
except Exception as e:
pass
else:
print("No replace handler for %s" % inputtype)
return liquiddata
skipkeys = [" "]
newoutput = liquiddata[:]
for pattern in patterns:
keylocations = []
parsedvalue = ""
record = False
index = -1
for key in liquiddata:
# Return instant if possible
if inputtype == "json":
try:
json.loads(newoutput)
return newoutput
except:
pass
index += 1
if not key:
if record:
keylocations.append(index)
parsedvalue += key
continue
if key in skipkeys:
if record:
keylocations.append(index)
parsedvalue += key
continue
if key == pattern[0] and not record:
record = True
if key not in pattern:
keylocations = []
parsedvalue = ""
record = False
if record:
keylocations.append(index)
parsedvalue += key
if len(parsedvalue) == 0:
continue
evaluated_value = parsedvalue[:]
for skipkey in skipkeys:
evaluated_value = "".join(evaluated_value.split(skipkey))
if evaluated_value == pattern:
#print("Found matching: %s (%s)" % (parsedvalue, keylocations))
#print("Should replace with: %s" % patterns[pattern])
newoutput = newoutput.replace(parsedvalue, patterns[pattern], -1)
# Return instant if possible
if inputtype == "json":
try:
json.loads(newoutput)
return newoutput
except:
pass
for pattern in regex_patterns:
newlines = []
for line in newoutput.split("\n"):
replaced_line = re.sub(pattern, regex_patterns[pattern], line)
newlines.append(replaced_line)
newoutput = "\n".join(newlines)
# Return instant if possible
if inputtype == "json":
try:
json.loads(newoutput)
return newoutput
except:
pass
# Dont return json properly unless actually json
if inputtype == "json":
try:
json.loads(newoutput)
return newoutput
except:
# Returns original if json fixing didn't work
return liquiddata
return newoutput
print("Start:\n%s" % input_data)
try:
newinput = patternfix_string(input_data,
{
"{{|": '{{ "" |',
},
{
#r'\{\{\s*|': "{{ '' |",
r'\{\{\s*\$[^|}]+\s*\|': '{{ "" |',
}
,
inputtype="liquid"
)
except Exception as e:
print("[ERROR} Failed liquid parsing fix: %s" % e)
newinput = input_data
try:
newinput = patternfix_string(newinput,
{
},
{
r'\"\s*\:\s*,': '\": "",',
r'\"\s*\:\s*\$[^,]+\w*\,': '\": "",',
}
,
inputtype="json"
)
try:
json.loads(newinput)
print("It's json! Override.")
except Exception as e:
print("Bad json. DONT use the value at all: %s" % e)
except Exception as e:
print("[ERROR} Failed json parsing fix: %s" % e)
print("\nEnd:\n%s" % newinput)
backend/app_sdk/build.sh
-51
View File
@@ -1,51 +0,0 @@
#!/bin/bash
### DEFAULT
NAME=shuffle-app_sdk
VERSION=1.2.0
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly
docker push frikky/shuffle:app_sdk
docker push ghcr.io/frikky/$NAME:$VERSION
docker push ghcr.io/frikky/$NAME:nightly
docker push ghcr.io/frikky/$NAME:latest
docker push shuffle/shuffle:app_sdk
docker push ghcr.io/shuffle/$NAME:$VERSION
docker push ghcr.io/shuffle/$NAME:nightly
docker push ghcr.io/shuffle/$NAME:latest
#### UBUNTU
NAME=shuffle-app_sdk_ubuntu
docker build . -f Dockerfile_ubuntu -t frikky/shuffle:app_sdk_ubuntu -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
docker push frikky/shuffle:app_sdk_ubuntu
docker push ghcr.io/frikky/$NAME:$VERSION
#### Alpine GRPC
NAME=shuffle-app_sdk_grpc
docker build . -f Dockerfile_alpine_grpc -t frikky/shuffle:app_sdk_grpc -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
docker push frikky/shuffle:app_sdk_grpc
docker push ghcr.io/frikky/$NAME:$VERSION
#### KALI ###
#NAME=shuffle-app_sdk_kali
#docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
#
#docker push frikky/shuffle:app_sdk_kali
#docker push ghcr.io/frikky/$NAME:$VERSION
#docker push ghcr.io/frikky/$NAME:nightly
### BLACKARCH ###
#NAME=shuffle-app_sdk_blackarch
#docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
#
#docker push frikky/shuffle:app_sdk_blackarch
#docker push ghcr.io/frikky/$NAME:$VERSION
#docker push ghcr.io/frikky/$NAME:nightly
backend/app_sdk/recurse_test.py
-326
View File
@@ -1,326 +0,0 @@
## A test script of the recurse_json function
## to validate that it can handle the different types of data
## and follow the dot formation format
import re
import json
def recurse_json(basejson, parsersplit):
match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?"
try:
outercnt = 0
# Loops over split values
splitcnt = -1
for value in parsersplit:
splitcnt += 1
#if " " in value:
# value = value.replace(" ", "_", -1)
actualitem = re.findall(match, value, re.MULTILINE)
# Goes here if loop
if value == "#":
newvalue = []
if basejson == None:
return "", False
for innervalue in basejson:
# 1. Check the next item (message)
# 2. Call this function again
try:
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:])
except IndexError:
# Only in here if it's the last loop without anything in it?
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:])
newvalue.append(ret)
# Magical way of returning which makes app sdk identify
# it as multi execution
return newvalue, True
# Checks specific regex like #1-2 for index 1-2 in a loop
elif len(actualitem) > 0:
is_loop = True
newvalue = []
firstitem = actualitem[0][0]
seconditem = actualitem[0][1]
if isinstance(firstitem, int):
firstitem = str(firstitem)
if isinstance(seconditem, int):
seconditem = str(seconditem)
#print("[DEBUG] ACTUAL PARSED: %s" % actualitem)
# Means it's a single item -> continue
if seconditem == "":
#print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson)))
if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end":
firstitem = len(basejson)-1
elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first":
firstitem = 0
else:
firstitem = int(firstitem)
#print(f"[DEBUG] Post lower checks with item {firstitem}")
tmpitem = basejson[int(firstitem)]
try:
newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
except IndexError:
newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
else:
#print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem))
if isinstance(firstitem, str):
if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end":
firstitem = len(basejson)-1
elif firstitem.lower() == "min" or firstitem.lower() == "first":
firstitem = 0
else:
firstitem = int(firstitem)
else:
firstitem = int(firstitem)
if isinstance(seconditem, str):
if str(seconditem).lower() == "max" or str(seconditem).lower() == "last" or str(firstitem).lower() == "end":
seconditem = len(basejson)-1
elif str(seconditem).lower() == "min" or str(seconditem).lower() == "first":
seconditem = 0
else:
seconditem = int(seconditem)
else:
seconditem = int(seconditem)
#print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}")
newvalue = []
if int(seconditem) > len(basejson):
seconditem = len(basejson)
for i in range(int(firstitem), int(seconditem)+1):
# 1. Check the next item (message)
# 2. Call this function again
try:
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:])
except IndexError:
#print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt])
#ret = innervalue
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:])
newvalue.append(ret)
return newvalue, is_loop
else:
if len(value) == 0:
return basejson, False
try:
if isinstance(basejson, list):
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
return basejson, False
elif isinstance(basejson, bool):
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
return basejson, False
elif isinstance(basejson, int):
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
return basejson, False
elif isinstance(basejson[value], str):
try:
if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")):
basejson = json.loads(basejson[value])
else:
# Should we sanitize here?
#print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value]))
# Check if we are on the last item?
if outercnt == len(parsersplit)-1:
#print("[DEBUG] LAST KEY")
return str(basejson[value]), False
else:
#print("[DEBUG] NOT LAST KEY")
pass
except json.decoder.JSONDecodeError as e:
return str(basejson[value]), False
else:
basejson = basejson[value]
except KeyError as e:
print("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e))
if "_" in value:
value = value.replace("_", " ", -1)
elif " " in value:
value = value.replace(" ", "_", -1)
try:
if isinstance(basejson, list):
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
return basejson, False
elif isinstance(basejson, bool):
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
return basejson, False
elif isinstance(basejson, int):
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
return basejson, False
elif isinstance(basejson[value], str):
#print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value])
try:
#print("[DEBUG] BASEJSON: %s" % basejson)
if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")):
basejson = json.loads(basejson[value])
else:
if outercnt == len(parsersplit)-1:
#print("LAST KEY (2)")
return str(basejson[value]), False
else:
#print("NOT LAST KEY (2)")
pass
except json.decoder.JSONDecodeError as e:
#print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value])
return str(basejson[value]), False
else:
basejson = basejson[value]
except KeyError as e:
# Check if previous key was handled or not
previouskey = parsersplit[outercnt-1]
#print("[DEBUG] PREVIOUS KEY: ", previouskey)
tmpval = previouskey + "." + value
#print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e))
if tmpval in basejson:
return basejson[tmpval], False
try:
currentsplitcnt = splitcnt
recursed_value = value
handled = False
#tmpbase = basejson
previouskey = value
while True:
#print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt)
newvalue = parsersplit[currentsplitcnt+1]
if newvalue == "#" or newvalue == "":
break
recursed_value += "." + newvalue
#print("\n\nRECURSED: ", recursed_value)
found = False
for key, value in basejson.items():
if recursed_value.lower() in key.lower():
found = True
if found == False:
#print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value)
# Check if we are on the last key or not
return "", False
#if outercnt == len(parsersplit)-1:
# print("[DEBUG] LAST KEY (3)")
# break
#else:
# print("[DEBUG] NOT LAST KEY (3)")
# return "", False
if recursed_value in basejson:
#print("[INFO] FOUND RECURSED VALUE: ", recursed_value)
basejson = basejson[recursed_value]
# Whether to dig deeper or not
if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str):
handled = False
else:
handled = True
break
currentsplitcnt += 1
if handled:
continue
break
except IndexError as e:
print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt])
return "", False
outercnt += 1
except KeyError as e:
print("[INFO] Lower keyerror: %s" % e)
return "", False
except Exception as e:
print("[WARNING] Exception: %s" % e)
return "", False
return basejson, False
print("[INFO] Starting")
#input_data = "test"
#input_data = "test2.data"
# Matchwith
basejson = {
"test": "hello",
"test2": {
"test3": "hello2",
"test3.data": "hello3",
"test4.data.testing": {
"value": "hello4"
},
"test5.data.hello": "wut",
},
"test3": ["hello", "hello2", "hello3"],
"test4": [{
"id": "1",
}]
}
# Inputexamples (ALL should be True)
inputs = {
#"": "",
"badkey": "",
"test": "hello",
"test2.badkey": "",
"test2.test3": "hello2",
"test2.test3.data": "hello3",
"test2.test4.data.testing": "{'value': 'hello4'}", # FIXME: Doesn't work due to break vs return "", False in last exception
"test2.test4.data.testing.value": "hello4", # FIXME: Doesn't work due to break vs return "", False in last exception. Not fixed as we didn't find one of these yet.
"test2.test5.data.hello": "wut",
"test2.test5.data.badkey": "",
"test3.#1": "hello2",
"test4.#0.id": "1",
"test4.#1.id": "",
}
outputs = []
for key, value in inputs.items():
parsersplit = key.split(".")
ret, is_loop = recurse_json(basejson, parsersplit)
print("\n\nOUTPUT RET (%s): %s" % (key, ret))
outputs.append("[%s]: %s = '%s' vs '%s'" % (str(ret) == str(value), key, ret, value))
print("\n\n%s" % "\n".join(outputs))
#input_data = ""
#input_data = "badkey"
#input_data = "test"
#input_data = "test2.data"
#input_data = "test2.test3.data"
#input_data = "test2.test4.data.testing.value.as"
#input_data = "test2.test5.data.hello"
backend/app_sdk/requirements.txt
+1
View File
@@ -6,3 +6,4 @@ flask[async]==2.0.2
waitress==2.1.0
#flask==1.1.2
python-dateutil==2.8.1