marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into launch

Browse Source
This commit is contained in:
Frikky
2020-10-14 12:36:56 +02:00
committed by GitHub
parent b4976e32b3 d38ee59aa5
commit 8fa39b5a42
15 changed files with 179 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+15
View File
@@ -0,0 +1,15 @@
**/.git*
**/.gitlab-ci.yml
**/.env
**/.dockerignore
**/Dockerfile*
**/node_modules
buildSrc/libs
.gradle/
build/
**build/
.idea/
!.idea/codeStyles/codeStyleConfig.xml
.DS_Store
*.log
out/
.env
+4
View File
@@ -34,3 +34,7 @@ DB_LOCATION=./shuffle-database
SHUFFLE_HTTP_PROXY=
SHUFFLE_HTTPS_PROXY=
SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.6.0"
backend/app_gen/openapi/test.go
+1 -1
View File
@@ -180,7 +180,7 @@ func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
api.Name = swagger.Info.Title
api.Description = swagger.Info.Description
api.IsValid = true
api.Link = swagger.Servers[0].URL // host doesnt exist lol
api.Link = swagger.Servers[0].URL // host does not exist lol
api.AppVersion = "1.0.0"
api.Environment = "cloud"
api.ID = ""
backend/app_gen/openapi/testGCP.go
+1 -1
View File
@@ -216,7 +216,7 @@ func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
api.Name = swagger.Info.Title
api.Description = swagger.Info.Description
api.IsValid = true
api.Link = swagger.Servers[0].URL // host doesnt exist lol
api.Link = swagger.Servers[0].URL // host does not exist lol
api.AppVersion = "1.0.0"
api.Environment = "cloud"
api.ID = ""
backend/app_gen/python-lib/generator.py
+1 -1
View File
@@ -246,7 +246,7 @@ def loop_modules(modules, data):
param["position"] = cnt
if not found:
# FIXME - waht here?
# FIXME - what here?
pass
#print("HANDLE NOT FOUND")
#print(param)
backend/app_sdk/app_base.py
+2 -2
View File
@@ -150,7 +150,7 @@ class AppBase:
self.logger.info("AFTER FULLEXEC stream result")
# Gets the value at the paranthesis level you want
# Gets the value at the parenthesis level you want
def parse_nested_param(string, level):
"""
Generate strings contained in nested (), indexing i = level
@@ -173,7 +173,7 @@ class AppBase:
except IndexError:
return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]]
# Finds the deepest level paranthesis in a string
# Finds the deepest level parenthesis in a string
def maxDepth(S):
current_max = 0
max = 0
backend/go-app/codegen.go
+1 -1
View File
@@ -425,7 +425,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
//uuid.NewV4().String()
api.IsValid = true
api.Link = swagger.Servers[0].URL // host doesnt exist lol
api.Link = swagger.Servers[0].URL // host does not exist lol
if strings.HasSuffix(api.Link, "/") {
api.Link = api.Link[:len(api.Link)-1]
}
backend/go-app/main.go
+2 -2
View File
@@ -6838,7 +6838,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
}
}
func init() {
func initHandlers() {
var err error
ctx := context.Background()
@@ -6964,7 +6964,7 @@ func init() {
// Had to move away from mux, which means Method is fucked up right now.
func main() {
//init()
initHandlers()
hostname, err := os.Hostname()
if err != nil {
hostname = "MISSING"
backend/go-app/walkoff.go
+101 -8
View File
@@ -28,6 +28,7 @@ import (
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/storage/memory"
http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
@@ -1551,6 +1552,12 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
log.Printf("Failed to increase total workflows: %s", err)
}
// recalculate authenticators stats
err = recalculateAppAuthentications()
if err != nil {
log.Printf("Authentications recalculation failed: %s", err)
}
//memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId)
//memcache.Delete(ctx, memcacheName)
//memcacheName = fmt.Sprintf("%s_workflows", user.Username)
@@ -4384,6 +4391,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation
func loadGithubWorkflows(url, username, password, userId, branch string) error {
fs := memfs.New()
// FIXME: add more git options lol
if strings.Contains(url, "github") || strings.Contains(url, "gitlab") || strings.Contains(url, "bitbucket") {
cloneOptions := &git.CloneOptions{
URL: url,
@@ -4392,16 +4400,11 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error {
// FIXME: Better auth.
if len(username) > 0 && len(password) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: username,
Password: password,
}
}
if len(branch) > 0 {
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
}
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
@@ -4409,9 +4412,31 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error {
return err
}
if len(branch) > 0 {
log.Printf("Checkout to branch: %s", branch)
w, _ := r.Worktree()
err := r.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"},
})
if err != nil {
log.Printf("Failed fetch for git repo: %s", err)
}
err = w.Checkout(&git.CheckoutOptions{
Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)),
Force: true,
})
if err != nil {
log.Printf("Failed checkout for git repo: %s", err)
return errors.New(fmt.Sprintf("Failed checking out to branch %s - does it exist?", branch))
}
}
dir, err := fs.ReadDir("/")
if err != nil {
log.Printf("FAiled reading folder: %s", err)
log.Printf("Failed reading folder: %s", err)
}
_ = r
@@ -4486,7 +4511,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
if err != nil {
log.Printf("Error with unmarshal tmpBody: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
resp.Write([]byte(`{"success": false, "reason": "json decode error"}`))
return
}
@@ -4495,14 +4520,82 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
if err != nil {
log.Printf("Failed to update workflows: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
// recalculate authenticators stats
err = recalculateAppAuthentications()
if err != nil {
log.Printf("Authentications recalculation failed: %s", err)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func recalculateAppAuthentications() error {
// create context
ctx := context.Background()
// form workflows list
workflows, err := getAllWorkflows(ctx)
if err != nil {
log.Printf("Error: Failed getting workflows: %s", err)
return err
}
// form authenticators list
auths, err := getAllWorkflowAppAuth(ctx)
if err != nil {
log.Printf("Error: Failed getting auths: %s", err)
return err
}
// iterate through auths
for _, auth := range auths {
// reset calculated values
auth.WorkflowCount = 0
auth.NodeCount = 0
auth.Usage = []AuthenticationUsage{}
// iterate through workflows to find which uses this auth
for _, workflow := range workflows {
hasCurrentAuth := false
usageItem := AuthenticationUsage{
WorkflowId: workflow.ID,
Nodes: []string{},
}
// iterate through actions
for _, action := range workflow.Actions {
if action.AuthenticationId == auth.Id {
// this workflow should be added to "usage" field
hasCurrentAuth = true
// add this action to list
usageItem.Nodes = append(usageItem.Nodes, action.ID)
}
}
// update current auth with found workflow
if hasCurrentAuth {
auth.WorkflowCount += 1
auth.NodeCount += int64(len(usageItem.Nodes))
auth.Usage = append(auth.Usage, usageItem)
}
}
// update record in database
err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id)
if err != nil {
log.Printf("Failed setting up app auth %s: %s", auth.Id, err)
}
}
return nil
}
func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
docker-compose.yml
+3
View File
@@ -61,6 +61,9 @@ services:
- HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600
- SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
- SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
- SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
restart: unless-stopped
database:
#build: ./backend/database
frontend/src/views/Admin.jsx
+2 -2
View File
@@ -1159,7 +1159,7 @@ const Admin = (props) => {
console.log("Show apps with this category")
}}
>
Find app ({data.apps.length})
Find app ({data.apps === null ? 0 : data.apps.length})
</Button>
</ListItemText>
</ListItem>
@@ -1222,7 +1222,7 @@ const Admin = (props) => {
style={{minWidth: 150, maxWidth: 150}}
/>
<ListItemText
primary={data.usage.length}
primary={data.usage === null ? 0 : data.usage.length}
style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
/>
<ListItemText
frontend/src/views/EditWebhook.jsx
+1 -1
View File
@@ -336,7 +336,7 @@ const EditWebhook = (props) => {
</div>
: null
// FIXME - needs refresh everytime you add a new workflow
// FIXME - needs refresh every time you add a new workflow
const workflowdata = Object.getOwnPropertyNames(webhookData).length > 0 && selectedWorkflows.length > 0 ?
<EditWorkflow globalUrl={globalUrl} inputworkflows={selectedWorkflows} inputname={webhookData.info.name} inputtype={webhookData.type} /> : null
frontend/src/views/Workflows.jsx
+22
View File
@@ -65,6 +65,7 @@ const Workflows = (props) => {
const [field1, setField1] = React.useState("")
const [field2, setField2] = React.useState("")
const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/frikky/shuffle-workflows")
const [downloadBranch, setDownloadBranch] = React.useState("master")
const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false)
const [modalOpen, setModalOpen] = React.useState(false);
@@ -1186,6 +1187,7 @@ const Workflows = (props) => {
const parsedData = {
"url": url,
"field_3": downloadBranch || 'master'
}
if (field1.length > 0) {
@@ -1280,6 +1282,26 @@ const Workflows = (props) => {
fullWidth
/>
<span style={{marginTop: 10}}>Branch (default value is "master"):</span>
<div style={{display: "flex"}}>
<TextField
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
value={downloadBranch}
InputProps={{
style:{
color: "white",
height: "50px",
fontSize: "1em",
},
}}
onChange={e => setDownloadBranch(e.target.value)}
placeholder="master"
fullWidth
/>
</div>
<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
<div style={{display: "flex"}}>
<TextField
functions/onprem/orborus/orborus.go
+18 -5
View File
@@ -36,7 +36,10 @@ var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
//var baseimagename = "docker.pkg.github.com/frikky/shuffle"
//var baseimagename = "ghcr.io/frikky"
var baseimagename = "frikky/shuffle"
// var baseimagename = "frikky/shuffle"
var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME")
var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX")
var orgId = os.Getenv("ORG_ID")
var baseUrl = os.Getenv("BASE_URL")
@@ -213,10 +216,19 @@ func initializeImages() {
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
// check whether theyre the same first
if baseimageregistry == "" {
baseimageregistry = "docker.io"
}
if baseimagename == "" {
baseimagename = "frikky/shuffle"
}
// check whether they are the same first
images := []string{
fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
fmt.Sprintf("docker.io/%s:worker", baseimagename),
fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix),
fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix),
// fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
// fmt.Sprintf("docker.io/%s:worker", baseimagename),
//fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion),
//fmt.Sprintf("%s/app_sdk:%s", baseimagename, appSdkVersion),
@@ -284,7 +296,8 @@ func main() {
//workerVersion := "0.1.0"
//workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion)
//workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion)
workerImage := fmt.Sprintf("docker.io/%s:worker", baseimagename)
// workerImage := fmt.Sprintf("docker.io/%s:worker", baseimagename)
workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix)
log.Printf("[INFO] Finished configuring docker environment")
functions/onprem/worker/Dockerfile
+5
View File
@@ -10,6 +10,11 @@ COPY worker.go /app/worker.go
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
FROM alpine:3.12
ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.6.0
RUN apk add --no-cache bash
COPY --from=builder /app/ /