Moving all sdk files to its own directory under MIT
This commit is contained in:
@@ -1,21 +0,0 @@
|
|||||||
#FROM python:3.9.1-alpine as base
|
|
||||||
FROM python:3.10.0-alpine as base
|
|
||||||
#FROM python:3.11.3-alpine as base
|
|
||||||
|
|
||||||
FROM base as builder
|
|
||||||
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils
|
|
||||||
|
|
||||||
RUN mkdir /install
|
|
||||||
WORKDIR /install
|
|
||||||
|
|
||||||
FROM base
|
|
||||||
|
|
||||||
#--no-cache
|
|
||||||
RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils
|
|
||||||
|
|
||||||
COPY --from=builder /install /usr/local
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip3 install -r /requirements.txt
|
|
||||||
|
|
||||||
COPY __init__.py /app/walkoff_app_sdk/__init__.py
|
|
||||||
COPY app_base.py /app/walkoff_app_sdk/app_base.py
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
FROM python:3.10.0-alpine as base
|
|
||||||
|
|
||||||
FROM base as builder
|
|
||||||
RUN apk --no-cache add --update \
|
|
||||||
alpine-sdk \
|
|
||||||
build-base \
|
|
||||||
g++ \
|
|
||||||
gcc \
|
|
||||||
libffi \
|
|
||||||
libffi-dev \
|
|
||||||
libstdc++ \
|
|
||||||
linux-headers \
|
|
||||||
musl-dev \
|
|
||||||
openssl-dev \
|
|
||||||
tzdata \
|
|
||||||
coreutils
|
|
||||||
|
|
||||||
RUN pip install --upgrade pip && \
|
|
||||||
pip install --prefix="/install" --no-cache-dir grpcio grpcio-tools && \
|
|
||||||
apk del --purge \
|
|
||||||
g++ \
|
|
||||||
gcc \
|
|
||||||
musl-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libstdc++ \
|
|
||||||
build-base \
|
|
||||||
linux-headers
|
|
||||||
|
|
||||||
RUN mkdir -p /install
|
|
||||||
WORKDIR /install
|
|
||||||
|
|
||||||
FROM base
|
|
||||||
|
|
||||||
#--no-cache
|
|
||||||
RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev coreutils
|
|
||||||
|
|
||||||
COPY --from=builder /install /usr/local
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip3 install -r /requirements.txt
|
|
||||||
|
|
||||||
COPY __init__.py /app/walkoff_app_sdk/__init__.py
|
|
||||||
COPY app_base.py /app/walkoff_app_sdk/app_base.py
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
FROM blackarchlinux/blackarch as base
|
|
||||||
|
|
||||||
FROM base as builder
|
|
||||||
|
|
||||||
RUN /bin/pacman -Syu --noconfirm
|
|
||||||
|
|
||||||
RUN /bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y
|
|
||||||
|
|
||||||
RUN mkdir /install
|
|
||||||
WORKDIR /install
|
|
||||||
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip install --prefix="/install" -r /requirements.txt
|
|
||||||
|
|
||||||
FROM base
|
|
||||||
|
|
||||||
COPY --from=builder /install /usr/local
|
|
||||||
COPY __init__.py /app/walkoff_app_sdk/__init__.py
|
|
||||||
COPY app_base.py /app/walkoff_app_sdk/app_base.py
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
FROM kalilinux/kali-rolling as base
|
|
||||||
|
|
||||||
FROM base as builder
|
|
||||||
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get dist-upgrade -y
|
|
||||||
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
|
|
||||||
|
|
||||||
RUN mkdir /install
|
|
||||||
WORKDIR /install
|
|
||||||
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip install --prefix="/install" -r /requirements.txt
|
|
||||||
|
|
||||||
FROM base
|
|
||||||
|
|
||||||
COPY --from=builder /install /usr/local
|
|
||||||
COPY __init__.py /app/walkoff_app_sdk/__init__.py
|
|
||||||
COPY app_base.py /app/walkoff_app_sdk/app_base.py
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
FROM ubuntu as base
|
|
||||||
|
|
||||||
FROM base as builder
|
|
||||||
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get dist-upgrade -y
|
|
||||||
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
|
|
||||||
|
|
||||||
RUN mkdir /install
|
|
||||||
WORKDIR /install
|
|
||||||
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip install --prefix="/install" -r /requirements.txt
|
|
||||||
|
|
||||||
FROM base
|
|
||||||
RUN apt-get update
|
|
||||||
RUN apt-get dist-upgrade -y
|
|
||||||
RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
|
|
||||||
|
|
||||||
COPY --from=builder /install /usr/local
|
|
||||||
COPY __init__.py /app/walkoff_app_sdk/__init__.py
|
|
||||||
COPY app_base.py /app/walkoff_app_sdk/app_base.py
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2020 Frikkylikeme
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
# app_sdk.py
|
|
||||||
This is the SDK used for apps to behave like they should.
|
|
||||||
|
|
||||||
## If you want to update apps.. PS: downloads from docker hub do overrides.. :)
|
|
||||||
1. Write your code & check if runtime works
|
|
||||||
2. Build app_base image
|
|
||||||
3. docker rm $(docker ps -aq) # Remove all stopped containers
|
|
||||||
4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
|
|
||||||
5. Rebuild the Docker image (click load in GUI?)
|
|
||||||
|
|
||||||
## Cloud updates
|
|
||||||
1. Go to shuffle cloud on GCP
|
|
||||||
2. Go to Cloud Storage
|
|
||||||
3. Find shuffler.appspot.com
|
|
||||||
4. Navigate to generated_apps/baseline
|
|
||||||
5. Update SDK there. This will make all new apps run with the new SDK
|
|
||||||
|
|
||||||
## Cloud app force-updates
|
|
||||||
1. Run the "stitcher.go" program in the public shuffle-shared repository.
|
|
||||||
|
|
||||||
# LICENSE
|
|
||||||
Everything in here is MIT, not AGPLv3 as indicated by the license.
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,187 +0,0 @@
|
|||||||
import re
|
|
||||||
import json
|
|
||||||
|
|
||||||
input_data = """{
|
|
||||||
"test4": $test,
|
|
||||||
"test5": ,
|
|
||||||
"test6": "what"
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
input_data = """{
|
|
||||||
"test0": {{ '' | default: [] }},
|
|
||||||
"test": {{ | default: [] }},
|
|
||||||
"test2": {{ $test.asd | default: [] }},
|
|
||||||
"test3": {{ {"key": "val} | default: [] }},
|
|
||||||
"test4": $test,
|
|
||||||
"test5": ,
|
|
||||||
"test6": "what"
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
liquiddata = "{{ $test.asd | some other stuff {{ $test.xyz | more stuff"
|
|
||||||
pattern = r'\{\{\s*\$[^|}]+\s*\|'
|
|
||||||
|
|
||||||
replaced_data = re.sub(pattern, "{{ '' |", liquiddata)
|
|
||||||
print(replaced_data)
|
|
||||||
|
|
||||||
|
|
||||||
def patternfix_string(liquiddata, patterns, regex_patterns, inputtype="liquid"):
|
|
||||||
if not inputtype or inputtype == "liquid":
|
|
||||||
if "{{" not in liquiddata or "}}" not in liquiddata:
|
|
||||||
return liquiddata
|
|
||||||
elif inputtype == "json":
|
|
||||||
liquiddata = liquiddata.strip()
|
|
||||||
|
|
||||||
# Validating if it looks like json or not
|
|
||||||
if liquiddata[0] == "{" and liquiddata[len(liquiddata)-1] == "}":
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
if liquiddata[0] == "[" and liquiddata[len(liquiddata)-1] == "]":
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
return liquiddata
|
|
||||||
|
|
||||||
# If it's already json, don't touch it
|
|
||||||
try:
|
|
||||||
json.loads(liquiddata)
|
|
||||||
return liquiddata
|
|
||||||
except Exception as e:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
print("No replace handler for %s" % inputtype)
|
|
||||||
return liquiddata
|
|
||||||
|
|
||||||
skipkeys = [" "]
|
|
||||||
newoutput = liquiddata[:]
|
|
||||||
for pattern in patterns:
|
|
||||||
keylocations = []
|
|
||||||
parsedvalue = ""
|
|
||||||
record = False
|
|
||||||
index = -1
|
|
||||||
for key in liquiddata:
|
|
||||||
|
|
||||||
# Return instant if possible
|
|
||||||
if inputtype == "json":
|
|
||||||
try:
|
|
||||||
json.loads(newoutput)
|
|
||||||
return newoutput
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
index += 1
|
|
||||||
if not key:
|
|
||||||
if record:
|
|
||||||
keylocations.append(index)
|
|
||||||
parsedvalue += key
|
|
||||||
|
|
||||||
continue
|
|
||||||
|
|
||||||
if key in skipkeys:
|
|
||||||
if record:
|
|
||||||
keylocations.append(index)
|
|
||||||
parsedvalue += key
|
|
||||||
|
|
||||||
continue
|
|
||||||
|
|
||||||
if key == pattern[0] and not record:
|
|
||||||
record = True
|
|
||||||
|
|
||||||
if key not in pattern:
|
|
||||||
keylocations = []
|
|
||||||
parsedvalue = ""
|
|
||||||
record = False
|
|
||||||
|
|
||||||
if record:
|
|
||||||
keylocations.append(index)
|
|
||||||
parsedvalue += key
|
|
||||||
|
|
||||||
if len(parsedvalue) == 0:
|
|
||||||
continue
|
|
||||||
|
|
||||||
evaluated_value = parsedvalue[:]
|
|
||||||
for skipkey in skipkeys:
|
|
||||||
evaluated_value = "".join(evaluated_value.split(skipkey))
|
|
||||||
|
|
||||||
if evaluated_value == pattern:
|
|
||||||
#print("Found matching: %s (%s)" % (parsedvalue, keylocations))
|
|
||||||
#print("Should replace with: %s" % patterns[pattern])
|
|
||||||
|
|
||||||
newoutput = newoutput.replace(parsedvalue, patterns[pattern], -1)
|
|
||||||
|
|
||||||
# Return instant if possible
|
|
||||||
if inputtype == "json":
|
|
||||||
try:
|
|
||||||
json.loads(newoutput)
|
|
||||||
return newoutput
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
for pattern in regex_patterns:
|
|
||||||
newlines = []
|
|
||||||
for line in newoutput.split("\n"):
|
|
||||||
replaced_line = re.sub(pattern, regex_patterns[pattern], line)
|
|
||||||
newlines.append(replaced_line)
|
|
||||||
|
|
||||||
newoutput = "\n".join(newlines)
|
|
||||||
|
|
||||||
# Return instant if possible
|
|
||||||
if inputtype == "json":
|
|
||||||
try:
|
|
||||||
json.loads(newoutput)
|
|
||||||
return newoutput
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Dont return json properly unless actually json
|
|
||||||
if inputtype == "json":
|
|
||||||
try:
|
|
||||||
json.loads(newoutput)
|
|
||||||
return newoutput
|
|
||||||
except:
|
|
||||||
# Returns original if json fixing didn't work
|
|
||||||
return liquiddata
|
|
||||||
|
|
||||||
return newoutput
|
|
||||||
|
|
||||||
print("Start:\n%s" % input_data)
|
|
||||||
|
|
||||||
try:
|
|
||||||
newinput = patternfix_string(input_data,
|
|
||||||
{
|
|
||||||
"{{|": '{{ "" |',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
#r'\{\{\s*|': "{{ '' |",
|
|
||||||
r'\{\{\s*\$[^|}]+\s*\|': '{{ "" |',
|
|
||||||
}
|
|
||||||
,
|
|
||||||
inputtype="liquid"
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
print("[ERROR} Failed liquid parsing fix: %s" % e)
|
|
||||||
newinput = input_data
|
|
||||||
|
|
||||||
try:
|
|
||||||
newinput = patternfix_string(newinput,
|
|
||||||
{
|
|
||||||
},
|
|
||||||
{
|
|
||||||
r'\"\s*\:\s*,': '\": "",',
|
|
||||||
r'\"\s*\:\s*\$[^,]+\w*\,': '\": "",',
|
|
||||||
}
|
|
||||||
,
|
|
||||||
inputtype="json"
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
json.loads(newinput)
|
|
||||||
print("It's json! Override.")
|
|
||||||
except Exception as e:
|
|
||||||
print("Bad json. DONT use the value at all: %s" % e)
|
|
||||||
except Exception as e:
|
|
||||||
print("[ERROR} Failed json parsing fix: %s" % e)
|
|
||||||
|
|
||||||
print("\nEnd:\n%s" % newinput)
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
### DEFAULT
|
|
||||||
NAME=shuffle-app_sdk
|
|
||||||
VERSION=1.2.0
|
|
||||||
|
|
||||||
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
|
|
||||||
docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly -t shuffle/shuffle:app_sdk -t shuffle/$NAME:$VERSION -t docker.pkg.github.com/shuffle/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:$VERSION -t ghcr.io/shuffle/$NAME:nightly
|
|
||||||
|
|
||||||
docker push frikky/shuffle:app_sdk
|
|
||||||
docker push ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
docker push ghcr.io/frikky/$NAME:nightly
|
|
||||||
docker push ghcr.io/frikky/$NAME:latest
|
|
||||||
|
|
||||||
docker push shuffle/shuffle:app_sdk
|
|
||||||
docker push ghcr.io/shuffle/$NAME:$VERSION
|
|
||||||
docker push ghcr.io/shuffle/$NAME:nightly
|
|
||||||
docker push ghcr.io/shuffle/$NAME:latest
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### UBUNTU
|
|
||||||
NAME=shuffle-app_sdk_ubuntu
|
|
||||||
docker build . -f Dockerfile_ubuntu -t frikky/shuffle:app_sdk_ubuntu -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
docker push frikky/shuffle:app_sdk_ubuntu
|
|
||||||
docker push ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
|
|
||||||
#### Alpine GRPC
|
|
||||||
NAME=shuffle-app_sdk_grpc
|
|
||||||
docker build . -f Dockerfile_alpine_grpc -t frikky/shuffle:app_sdk_grpc -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
docker push frikky/shuffle:app_sdk_grpc
|
|
||||||
docker push ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#### KALI ###
|
|
||||||
#NAME=shuffle-app_sdk_kali
|
|
||||||
#docker build . -f Dockerfile_kali -t frikky/shuffle:app_sdk_kali -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
#
|
|
||||||
#docker push frikky/shuffle:app_sdk_kali
|
|
||||||
#docker push ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
#docker push ghcr.io/frikky/$NAME:nightly
|
|
||||||
|
|
||||||
### BLACKARCH ###
|
|
||||||
#NAME=shuffle-app_sdk_blackarch
|
|
||||||
#docker build . -f Dockerfile_blackarch -t frikky/shuffle:app_sdk_blackarch -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
#
|
|
||||||
#docker push frikky/shuffle:app_sdk_blackarch
|
|
||||||
#docker push ghcr.io/frikky/$NAME:$VERSION
|
|
||||||
#docker push ghcr.io/frikky/$NAME:nightly
|
|
||||||
@@ -1,326 +0,0 @@
|
|||||||
## A test script of the recurse_json function
|
|
||||||
## to validate that it can handle the different types of data
|
|
||||||
## and follow the dot formation format
|
|
||||||
|
|
||||||
|
|
||||||
import re
|
|
||||||
import json
|
|
||||||
|
|
||||||
def recurse_json(basejson, parsersplit):
|
|
||||||
match = "#([0-9a-z]+):?-?([0-9a-z]+)?#?"
|
|
||||||
try:
|
|
||||||
outercnt = 0
|
|
||||||
|
|
||||||
# Loops over split values
|
|
||||||
splitcnt = -1
|
|
||||||
for value in parsersplit:
|
|
||||||
splitcnt += 1
|
|
||||||
#if " " in value:
|
|
||||||
# value = value.replace(" ", "_", -1)
|
|
||||||
|
|
||||||
actualitem = re.findall(match, value, re.MULTILINE)
|
|
||||||
# Goes here if loop
|
|
||||||
if value == "#":
|
|
||||||
newvalue = []
|
|
||||||
|
|
||||||
if basejson == None:
|
|
||||||
return "", False
|
|
||||||
|
|
||||||
for innervalue in basejson:
|
|
||||||
# 1. Check the next item (message)
|
|
||||||
# 2. Call this function again
|
|
||||||
|
|
||||||
try:
|
|
||||||
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:])
|
|
||||||
except IndexError:
|
|
||||||
# Only in here if it's the last loop without anything in it?
|
|
||||||
ret, is_loop = recurse_json(innervalue, parsersplit[outercnt:])
|
|
||||||
|
|
||||||
newvalue.append(ret)
|
|
||||||
|
|
||||||
# Magical way of returning which makes app sdk identify
|
|
||||||
# it as multi execution
|
|
||||||
return newvalue, True
|
|
||||||
|
|
||||||
# Checks specific regex like #1-2 for index 1-2 in a loop
|
|
||||||
elif len(actualitem) > 0:
|
|
||||||
|
|
||||||
is_loop = True
|
|
||||||
newvalue = []
|
|
||||||
firstitem = actualitem[0][0]
|
|
||||||
seconditem = actualitem[0][1]
|
|
||||||
if isinstance(firstitem, int):
|
|
||||||
firstitem = str(firstitem)
|
|
||||||
if isinstance(seconditem, int):
|
|
||||||
seconditem = str(seconditem)
|
|
||||||
|
|
||||||
#print("[DEBUG] ACTUAL PARSED: %s" % actualitem)
|
|
||||||
|
|
||||||
# Means it's a single item -> continue
|
|
||||||
if seconditem == "":
|
|
||||||
#print("[INFO] In first - handling %s. Len: %d" % (firstitem, len(basejson)))
|
|
||||||
if str(firstitem).lower() == "max" or str(firstitem).lower() == "last" or str(firstitem).lower() == "end":
|
|
||||||
firstitem = len(basejson)-1
|
|
||||||
elif str(firstitem).lower() == "min" or str(firstitem).lower() == "first":
|
|
||||||
firstitem = 0
|
|
||||||
else:
|
|
||||||
firstitem = int(firstitem)
|
|
||||||
|
|
||||||
#print(f"[DEBUG] Post lower checks with item {firstitem}")
|
|
||||||
tmpitem = basejson[int(firstitem)]
|
|
||||||
try:
|
|
||||||
newvalue, is_loop = recurse_json(tmpitem, parsersplit[outercnt+1:])
|
|
||||||
except IndexError:
|
|
||||||
newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:])
|
|
||||||
else:
|
|
||||||
#print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem))
|
|
||||||
if isinstance(firstitem, str):
|
|
||||||
if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end":
|
|
||||||
firstitem = len(basejson)-1
|
|
||||||
elif firstitem.lower() == "min" or firstitem.lower() == "first":
|
|
||||||
firstitem = 0
|
|
||||||
else:
|
|
||||||
firstitem = int(firstitem)
|
|
||||||
else:
|
|
||||||
firstitem = int(firstitem)
|
|
||||||
|
|
||||||
if isinstance(seconditem, str):
|
|
||||||
if str(seconditem).lower() == "max" or str(seconditem).lower() == "last" or str(firstitem).lower() == "end":
|
|
||||||
seconditem = len(basejson)-1
|
|
||||||
elif str(seconditem).lower() == "min" or str(seconditem).lower() == "first":
|
|
||||||
seconditem = 0
|
|
||||||
else:
|
|
||||||
seconditem = int(seconditem)
|
|
||||||
else:
|
|
||||||
seconditem = int(seconditem)
|
|
||||||
|
|
||||||
#print(f"[DEBUG] Post lower checks 2: {firstitem} AND {seconditem}")
|
|
||||||
newvalue = []
|
|
||||||
if int(seconditem) > len(basejson):
|
|
||||||
seconditem = len(basejson)
|
|
||||||
|
|
||||||
for i in range(int(firstitem), int(seconditem)+1):
|
|
||||||
# 1. Check the next item (message)
|
|
||||||
# 2. Call this function again
|
|
||||||
|
|
||||||
try:
|
|
||||||
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt+1:])
|
|
||||||
except IndexError:
|
|
||||||
#print("[DEBUG] INDEXERROR (1): ", parsersplit[outercnt])
|
|
||||||
#ret = innervalue
|
|
||||||
ret, tmp_loop = recurse_json(basejson[i], parsersplit[outercnt:])
|
|
||||||
|
|
||||||
newvalue.append(ret)
|
|
||||||
|
|
||||||
return newvalue, is_loop
|
|
||||||
|
|
||||||
else:
|
|
||||||
if len(value) == 0:
|
|
||||||
return basejson, False
|
|
||||||
|
|
||||||
try:
|
|
||||||
if isinstance(basejson, list):
|
|
||||||
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
|
|
||||||
return basejson, False
|
|
||||||
elif isinstance(basejson, bool):
|
|
||||||
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
|
|
||||||
return basejson, False
|
|
||||||
elif isinstance(basejson, int):
|
|
||||||
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
|
|
||||||
return basejson, False
|
|
||||||
elif isinstance(basejson[value], str):
|
|
||||||
try:
|
|
||||||
if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")):
|
|
||||||
basejson = json.loads(basejson[value])
|
|
||||||
else:
|
|
||||||
# Should we sanitize here?
|
|
||||||
#print("[DEBUG] VALUE TO SANITIZE FOR KEY '%s'?: %s" % (value, basejson[value]))
|
|
||||||
|
|
||||||
# Check if we are on the last item?
|
|
||||||
if outercnt == len(parsersplit)-1:
|
|
||||||
#print("[DEBUG] LAST KEY")
|
|
||||||
return str(basejson[value]), False
|
|
||||||
else:
|
|
||||||
#print("[DEBUG] NOT LAST KEY")
|
|
||||||
pass
|
|
||||||
|
|
||||||
except json.decoder.JSONDecodeError as e:
|
|
||||||
return str(basejson[value]), False
|
|
||||||
else:
|
|
||||||
basejson = basejson[value]
|
|
||||||
except KeyError as e:
|
|
||||||
print("[WARNING] Running secondary value check with replacement of underscore in %s: %s" % (value, e))
|
|
||||||
if "_" in value:
|
|
||||||
value = value.replace("_", " ", -1)
|
|
||||||
elif " " in value:
|
|
||||||
value = value.replace(" ", "_", -1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
if isinstance(basejson, list):
|
|
||||||
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (list): %s" % value)
|
|
||||||
return basejson, False
|
|
||||||
elif isinstance(basejson, bool):
|
|
||||||
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (bool): %s" % value)
|
|
||||||
return basejson, False
|
|
||||||
elif isinstance(basejson, int):
|
|
||||||
#print("[WARNING] VALUE IN ISINSTANCE IS NOT TO BE USED (int): %s" % value)
|
|
||||||
return basejson, False
|
|
||||||
elif isinstance(basejson[value], str):
|
|
||||||
#print(f"[INFO] LOADING STRING '%s' AS JSON" % basejson[value])
|
|
||||||
try:
|
|
||||||
#print("[DEBUG] BASEJSON: %s" % basejson)
|
|
||||||
if (basejson[value].endswith("}") and basejson[value].endswith("}")) or (basejson[value].startswith("[") and basejson[value].endswith("]")):
|
|
||||||
basejson = json.loads(basejson[value])
|
|
||||||
else:
|
|
||||||
|
|
||||||
if outercnt == len(parsersplit)-1:
|
|
||||||
#print("LAST KEY (2)")
|
|
||||||
return str(basejson[value]), False
|
|
||||||
else:
|
|
||||||
#print("NOT LAST KEY (2)")
|
|
||||||
pass
|
|
||||||
|
|
||||||
except json.decoder.JSONDecodeError as e:
|
|
||||||
#print("[DEBUG] RETURNING BECAUSE '%s' IS A NORMAL STRING (1)" % basejson[value])
|
|
||||||
return str(basejson[value]), False
|
|
||||||
else:
|
|
||||||
basejson = basejson[value]
|
|
||||||
except KeyError as e:
|
|
||||||
# Check if previous key was handled or not
|
|
||||||
previouskey = parsersplit[outercnt-1]
|
|
||||||
#print("[DEBUG] PREVIOUS KEY: ", previouskey)
|
|
||||||
|
|
||||||
tmpval = previouskey + "." + value
|
|
||||||
#print("\n\n[WARNING] Running third dot notation fix '%s' on data %s: %s" % (value, basejson, e))
|
|
||||||
if tmpval in basejson:
|
|
||||||
return basejson[tmpval], False
|
|
||||||
|
|
||||||
try:
|
|
||||||
currentsplitcnt = splitcnt
|
|
||||||
|
|
||||||
recursed_value = value
|
|
||||||
handled = False
|
|
||||||
|
|
||||||
#tmpbase = basejson
|
|
||||||
previouskey = value
|
|
||||||
while True:
|
|
||||||
#print("\n\n[DEBUG] CURRENTSPLITCNT: ", currentsplitcnt)
|
|
||||||
newvalue = parsersplit[currentsplitcnt+1]
|
|
||||||
if newvalue == "#" or newvalue == "":
|
|
||||||
break
|
|
||||||
|
|
||||||
recursed_value += "." + newvalue
|
|
||||||
#print("\n\nRECURSED: ", recursed_value)
|
|
||||||
|
|
||||||
found = False
|
|
||||||
for key, value in basejson.items():
|
|
||||||
if recursed_value.lower() in key.lower():
|
|
||||||
found = True
|
|
||||||
|
|
||||||
if found == False:
|
|
||||||
#print("[INFO] DIDN'T FIND similar VALUE: ", recursed_value)
|
|
||||||
|
|
||||||
# Check if we are on the last key or not
|
|
||||||
return "", False
|
|
||||||
#if outercnt == len(parsersplit)-1:
|
|
||||||
# print("[DEBUG] LAST KEY (3)")
|
|
||||||
# break
|
|
||||||
#else:
|
|
||||||
# print("[DEBUG] NOT LAST KEY (3)")
|
|
||||||
# return "", False
|
|
||||||
|
|
||||||
if recursed_value in basejson:
|
|
||||||
#print("[INFO] FOUND RECURSED VALUE: ", recursed_value)
|
|
||||||
basejson = basejson[recursed_value]
|
|
||||||
|
|
||||||
# Whether to dig deeper or not
|
|
||||||
if isinstance(basejson, bool) or isinstance(basejson, int) or isinstance(basejson, str):
|
|
||||||
handled = False
|
|
||||||
else:
|
|
||||||
handled = True
|
|
||||||
|
|
||||||
break
|
|
||||||
|
|
||||||
currentsplitcnt += 1
|
|
||||||
|
|
||||||
if handled:
|
|
||||||
continue
|
|
||||||
|
|
||||||
break
|
|
||||||
except IndexError as e:
|
|
||||||
print("[DEBUG] INDEXERROR (2):", parsersplit[outercnt])
|
|
||||||
return "", False
|
|
||||||
|
|
||||||
outercnt += 1
|
|
||||||
|
|
||||||
except KeyError as e:
|
|
||||||
print("[INFO] Lower keyerror: %s" % e)
|
|
||||||
return "", False
|
|
||||||
except Exception as e:
|
|
||||||
print("[WARNING] Exception: %s" % e)
|
|
||||||
return "", False
|
|
||||||
|
|
||||||
return basejson, False
|
|
||||||
|
|
||||||
print("[INFO] Starting")
|
|
||||||
|
|
||||||
#input_data = "test"
|
|
||||||
#input_data = "test2.data"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Matchwith
|
|
||||||
basejson = {
|
|
||||||
"test": "hello",
|
|
||||||
"test2": {
|
|
||||||
"test3": "hello2",
|
|
||||||
"test3.data": "hello3",
|
|
||||||
"test4.data.testing": {
|
|
||||||
"value": "hello4"
|
|
||||||
},
|
|
||||||
"test5.data.hello": "wut",
|
|
||||||
},
|
|
||||||
"test3": ["hello", "hello2", "hello3"],
|
|
||||||
"test4": [{
|
|
||||||
"id": "1",
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Inputexamples (ALL should be True)
|
|
||||||
inputs = {
|
|
||||||
#"": "",
|
|
||||||
"badkey": "",
|
|
||||||
"test": "hello",
|
|
||||||
"test2.badkey": "",
|
|
||||||
"test2.test3": "hello2",
|
|
||||||
"test2.test3.data": "hello3",
|
|
||||||
"test2.test4.data.testing": "{'value': 'hello4'}", # FIXME: Doesn't work due to break vs return "", False in last exception
|
|
||||||
"test2.test4.data.testing.value": "hello4", # FIXME: Doesn't work due to break vs return "", False in last exception. Not fixed as we didn't find one of these yet.
|
|
||||||
"test2.test5.data.hello": "wut",
|
|
||||||
"test2.test5.data.badkey": "",
|
|
||||||
"test3.#1": "hello2",
|
|
||||||
"test4.#0.id": "1",
|
|
||||||
"test4.#1.id": "",
|
|
||||||
}
|
|
||||||
|
|
||||||
outputs = []
|
|
||||||
for key, value in inputs.items():
|
|
||||||
parsersplit = key.split(".")
|
|
||||||
ret, is_loop = recurse_json(basejson, parsersplit)
|
|
||||||
print("\n\nOUTPUT RET (%s): %s" % (key, ret))
|
|
||||||
|
|
||||||
outputs.append("[%s]: %s = '%s' vs '%s'" % (str(ret) == str(value), key, ret, value))
|
|
||||||
|
|
||||||
print("\n\n%s" % "\n".join(outputs))
|
|
||||||
|
|
||||||
#input_data = ""
|
|
||||||
#input_data = "badkey"
|
|
||||||
#input_data = "test"
|
|
||||||
#input_data = "test2.data"
|
|
||||||
#input_data = "test2.test3.data"
|
|
||||||
#input_data = "test2.test4.data.testing.value.as"
|
|
||||||
#input_data = "test2.test5.data.hello"
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
urllib3==1.26.18
|
|
||||||
requests==2.31.0
|
|
||||||
MarkupSafe==2.0.1
|
|
||||||
liquidpy==0.8.1
|
|
||||||
flask[async]==2.0.2
|
|
||||||
waitress==2.1.0
|
|
||||||
#flask==1.1.2
|
|
||||||
python-dateutil==2.8.1
|
|
||||||
PyJWT==2.9.0
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
from setuptools import setup, find_packages
|
|
||||||
|
|
||||||
setup(
|
|
||||||
name='shuffle_sdk', # Name of the package
|
|
||||||
version='0.0.1', # Version number
|
|
||||||
description='The SDK used for Shuffle',
|
|
||||||
long_description=open('README.md').read(),
|
|
||||||
long_description_content_type='text/markdown',
|
|
||||||
author='Fredrik Saito Odegaardstuen',
|
|
||||||
author_email='frikky@shuffler.io',
|
|
||||||
url='https://github.com/shuffle/shuffle',
|
|
||||||
packages=find_packages(),
|
|
||||||
install_requires=[
|
|
||||||
'requests',
|
|
||||||
],
|
|
||||||
classifiers=[
|
|
||||||
'Programming Language :: Python :: 3',
|
|
||||||
'License :: OSI Approved :: MIT License',
|
|
||||||
'Operating System :: OS Independent',
|
|
||||||
],
|
|
||||||
python_requires='>=3.7', # Specify Python version requirements
|
|
||||||
)
|
|
||||||
Reference in New Issue
Block a user