Rebuild Shuffle structure for docker image builds

.gitignore

@@ -12,6 +12,9 @@ backend/go-app/generated*
 functions/generated_apps
 *.zip
 
+*openapi-parsers/generated/*
+*openapi-parsers/other/*
+
 
 backend/onprem/app_sdk/apps
 *test.py

backend/Dockerfile

@@ -11,6 +11,11 @@ ADD ./go-app/codegen.go /app
 
 ADD ./go-app/go.mod /app
 
+# Required files for code generation
+ADD ./app_sdk/app_base.py /app_sdk
+ADD ./app_sdk/static_baseline.py /app_sdk
+ADD ./app_gen /app_gen
+
 RUN go get -v
 
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp .
@@ -20,14 +25,10 @@ FROM alpine:latest as certs
 RUN apk --update add ca-certificates
 
 from scratch
-RUN mkdir /app
-RUN mkdir /app_sdk
-
-# Required files for code generation
-ADD ./app_sdk/app_sdk.py /app_sdk
-ADD ./app_sdk/static_baseline.py /app_sdk
 
 COPY --from=builder /app/ /app
+COPY --from=builder /app_sdk/ /app_sdk
+COPY --from=builder /app_gen/ /app_gen
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
 WORKDIR /app

backend/README.md

@@ -1,61 +1,18 @@
-# Backend setup
+# Backend 
-1 Go to https://console.cloud.google.com/apis/credentials?project=shuffle-241517&folder&organizationId and get credentials
+This folder has all parts necessary for the backend to run locally and in Docker
-2. Move the file to current folder (or make step 3 be your download folder or w/e)
-3. export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/Shuffle-2a19ff64af66.json
 
-# Backend run testserver (appengine)
+## Structure
-1. Set up gcloud locally
+* go-app: 	The backend. Modify these to edit the backend API.
-```bash
+* database: The datastore database.
-dev_appserver.py go-app/ --port=5001 --host=0.0.0.0 --enable_host_checking=false
+* app_sdk: 	The app_sdk for apps. MIT licensed. 
-```
+* app_gen: 	Code used when generating docker images. MIT licensed
+* tests: 		A bunch of cronscripts. There are no real, good tests yet
 
-# Backend deploy
+## Development
-* I created a simple script that moves the data into your GOPATH and deploys for you. This will require more tests in the future. 
+Shuffle's backend is written in Go, with apps being python (for now). More about local development can be seen in the main README.
 
-# OpenAPI spec checks
+Running the backend:
-Paths: 
-* /path/{variablename}?queryvar= <-- variable
-* ^variablename needs to be part of parameters too. 
-* ^queryvar needs to be part of parameters too.
 ```
-parameters:
+cd go-app
-- name: variablename
+go run *.go
-  in: path
-  description: Blah blah
-  required: true/false
-  schema:
-    type: string
-	enum: [a, b, c] # <-- not necessary, but could be great
-- name: queryvar
-  in: query
-  description: blah blah
-  required: true/false
-  schema:
-    type: string
-	enum: [a, b, c]
-```
-* requestBody? Not in GET, DELETE & HEAD. Can consume JSON, XML, form data, plai ntext & others. Can use markdown for the description.  
-* Do I care about the response? Maybe :o
-
-```
-requestBody:
-  description: Optional kind of description
-  required: false/true
-  content:
-    application/json:
-	  schema:
-	    type: object
-		additionalProperties: true
-		properties:
-		  name:
-		    type: string
-		  fav_number:
-		    type: integer
-		  required:
-		    - name
-			- email
-	  encoding:
-	    color:
-		  style: form
-		  explode: false
 ```

backend/app_gen/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Frikkylikeme
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

backend/app_sdk/readme

@@ -1,8 +0,0 @@
-# App_sdk
-App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk
-
-# Added static:baseline.py
-It's used for python code generation and should be under MIT.
-
-# LICENSE
-Everything in this folder is MIT, not AGPLv3

backend/build.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+docker stop shuffle-backend
+docker rm shuffle-backend
+docker rmi frikky/shuffle:backend 
+
+docker build . -t frikky/shuffle:backend
+
+echo "Starting server"
+#docker run -it \
+#	-p 5001:5001 \
+#	-v /var/run/docker.sock:/var/run/docker.sock \
+#	--env DATASTORE_EMULATOR_HOST=192.168.3.6:8000 \
+#	frikky/shuffle:backend	

backend/deploy-backend.sh

@@ -1,8 +0,0 @@
-# Deploys to backend
-echo "Deploying to appengine."
-mkdir -p $GOPATH/src/github.com/frikky/shuffle
-cp -r go-app/* $GOPATH/src/github.com/frikky/shuffle
-cd $GOPATH/src/github.com/frikky/shuffle
-go build
-go test
-gcloud app deploy $GOPATH/src/github.com/frikky/shuffle/app.yaml

backend/go-app/codegen.go

@@ -218,7 +218,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
 
 	// adding md5 based on input data to not overwrite earlier data.
 	generatedPath := "generated"
-	subpath := "../../app_gen/openapi/"
+	subpath := "../app_gen/openapi/"
 	identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
 	appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
 
@@ -704,6 +704,9 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
 	optionalParameters := []WorkflowAppActionParameter{}
 	if len(path.Connect.Parameters) > 0 {
 		for _, param := range path.Connect.Parameters {
+			if param.Value.Schema == nil {
+				continue
+			}
 			curParam := WorkflowAppActionParameter{
 				Name:        param.Value.Name,
 				Description: param.Value.Description,
@@ -876,6 +879,9 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
 	optionalParameters := []WorkflowAppActionParameter{}
 	if len(path.Head.Parameters) > 0 {
 		for _, param := range path.Head.Parameters {
+			if param.Value.Schema == nil {
+				continue
+			}
 			curParam := WorkflowAppActionParameter{
 				Name:        param.Value.Name,
 				Description: param.Value.Description,
@@ -958,6 +964,9 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
 	optionalParameters := []WorkflowAppActionParameter{}
 	if len(path.Delete.Parameters) > 0 {
 		for _, param := range path.Delete.Parameters {
+			if param.Value.Schema == nil {
+				continue
+			}
 			curParam := WorkflowAppActionParameter{
 				Name:        param.Value.Name,
 				Description: param.Value.Description,
@@ -1052,6 +1061,9 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
 	}
 	if len(path.Post.Parameters) > 0 {
 		for _, param := range path.Post.Parameters {
+			if param.Value.Schema == nil {
+				continue
+			}
 			curParam := WorkflowAppActionParameter{
 				Name:        param.Value.Name,
 				Description: param.Value.Description,
@@ -1145,6 +1157,9 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
 	}
 	if len(path.Patch.Parameters) > 0 {
 		for _, param := range path.Patch.Parameters {
+			if param.Value.Schema == nil {
+				continue
+			}
 			curParam := WorkflowAppActionParameter{
 				Name:        param.Value.Name,
 				Description: param.Value.Description,
@@ -1238,6 +1253,9 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
 	}
 	if len(path.Put.Parameters) > 0 {
 		for _, param := range path.Put.Parameters {
+			if param.Value.Schema == nil {
+				continue
+			}
 			curParam := WorkflowAppActionParameter{
 				Name:        param.Value.Name,
 				Description: param.Value.Description,

backend/go-app/main.go

@@ -5698,7 +5698,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
 	log.Println(len(user.PrivateApps))
 	c, err := request.Cookie("session_token")
 	if err == nil {
-		log.Printf("Should've deleted cache for %s with token %s", user.Username, c.Value)
+		//log.Printf("Should've deleted cache for %s with token %s", user.Username, c.Value)
 		//err = memcache.Delete(request.Context(), c.Value)
 		//err = memcache.Delete(request.Context(), user.ApiKey)
 	}

backend/run.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-# docker stop nginx
-# docker rm nginx
-# docker rmi nginx
-# 
-# echo "Running build for website"
-# sudo npm run build
-# docker build . -t nginx
-
-echo "Starting server"
-docker run -it \
-	-p 5001:5001 \
-	-v /var/run/docker.sock:/var/run/docker.sock \
-	--env DATASTORE_EMULATOR_HOST=192.168.3.6:8000 \
-	frikky/shuffle:backend	

backend/webhook/Dockerfile

@@ -1,15 +0,0 @@
-# Build environment
-# production environment
-from golang as builder
-
-RUN go get github.com/gorilla/handlers 
-RUN go get github.com/gorilla/mux 
-
-WORKDIR /app
-COPY webhook.go /app/webhook.go
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webhook .
-
-from scratch
-COPY --from=builder /app/ /
-
-CMD ["./webhook"]

backend/webhook/README.md

@@ -1,7 +0,0 @@
-# Steps to deploy to Google cloud function
-1. 
-```bash
-zip webhook.zip * 
-```
-2. Go to google cloud bucket and upload the zip
-3. Go to worker for webhook and upload

backend/webhook/functionhook.go

@@ -1,44 +0,0 @@
-package function
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"time"
-)
-
-// GetUserDetails - Get one user's details from randomuser.me API
-func GetUserDetails(w http.ResponseWriter, r *http.Request) {
-	randomUserClient := http.Client{
-		Timeout: time.Second * 3,
-	}
-
-	req, err := http.NewRequest(http.MethodGet, "https://randomuser.me/api/", nil)
-	if err != nil {
-		log.Fatal(err)
-		return
-	}
-
-	res, err2 := randomUserClient.Do(req)
-	if err2 != nil {
-		log.Fatal(err2)
-		return
-	}
-
-	body, err3 := ioutil.ReadAll(res.Body)
-	if err3 != nil {
-		log.Fatal(err3)
-	}
-
-	var o map[string]interface{}
-	json.Unmarshal([]byte(body), &o)
-
-	results := o["results"].([]interface{})
-	result := results[0].(map[string]interface{})
-
-	result["generator"] = "google-cloud-function"
-
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(result)
-}

backend/webhook/gcp_run.sh

@@ -1,4 +0,0 @@
-docker build . -t gcr.io/shuffle-241517/webhook
-docker push gcr.io/shuffle-241517/webhook
-
-gcloud beta run deploy webhook --image gcr.io/shuffle-241517/webhook

backend/webhook/run.sh

@@ -1,16 +0,0 @@
-docker stop webhook
-docker rm webhook
-docker rmi webhook
-
-docker build . -t webhook
-docker run -d \
-	-e "HOOKPORT=5001" \
-	-e "URIPATH=/webhook" \
-	-e "CALLBACKURL=http://192.168.3.6:5001/api/v1/hooks/d6ef8912e8bd37776e654cbc14c2629c/result" \
-	-p 6000:6000 \
-	--name webhook \
-	-h webhook \
-	--restart always \
-	webhook
-
-docker logs -f webhook

backend/webhook/webhook.go

@@ -1,260 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
-)
-
-type Info struct {
-	Url         string `json:"url" datastore:"url"`
-	Name        string `json:"name" datastore:"name"`
-	Description string `json:"description" datastore:"description"`
-}
-
-// Actions to be done by webhooks etc
-// Field is the actual field to use from json
-type HookAction struct {
-	Type  string `json:"type" datastore:"type"`
-	Name  string `json:"name" datastore:"name"`
-	Id    string `json:"id" datastore:"id"`
-	Field string `json:"field" datastore:"field"`
-}
-
-type Hook struct {
-	Id         string       `json:"id" datastore:"id"`
-	Info       Info         `json:"info" datastore:"info"`
-	Transforms struct{}     `json:"transforms" datastore:"transforms"`
-	Actions    []HookAction `json:"actions" datastore:"actions"`
-	Type       string       `json:"type" datastore:"type"`
-	Status     string       `json:"status" datastore:"status"`
-	Running    bool         `json:"running" datastore:"running"`
-}
-
-var hook Hook
-
-func handleWorkflowAction(request *http.Request, action HookAction) error {
-	//log.Printf("WORKFLOW!: %#v", action)
-	log.Printf("Should execute workflow %s", action.Id)
-
-	callbackUrl := os.Getenv("CALLBACKURL")
-	apikey := os.Getenv("APIKEY")
-	fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, action.Id)
-
-	// ret = requests.post(fullurl, headers=headers, json=data)
-	//if ret.status_code != 202:
-	//	print(ret.text)
-	//	print(ret.status_code)
-	//	print("Exiting workflows - run queue")
-	//	exit()
-
-	body, err := ioutil.ReadAll(request.Body)
-	if err != nil {
-		return err
-	}
-
-	// Execute a workflow
-	client := &http.Client{}
-	req, err := http.NewRequest(
-		"POST",
-		fullUrl,
-		bytes.NewBuffer(body),
-	)
-
-	if err != nil {
-		log.Printf("Error making http request: %s", req)
-		return err
-	}
-
-	req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
-	req.Header.Add("Content-Type", "application/json")
-	resp, err := client.Do(req)
-	if err != nil {
-		log.Printf("Error in http request: %s", req)
-	}
-
-	log.Printf("%#v", resp)
-	return nil
-}
-
-// FIXME - refresh hook information once in a while. Compare timestamps or something
-func callback(resp http.ResponseWriter, request *http.Request) {
-	//apikey = os.Getenv("APIKEY")
-	//hookId = os.Getenv("HOOKID")
-
-	handledWorkflowIds := []string{}
-	for _, item := range hook.Actions {
-		if item.Type == "" {
-			log.Printf("CONTINUE AAS EMPTY ITEM: %#v", item)
-			continue
-		}
-
-		if item.Type == "workflow" {
-			found := false
-			for _, workflowId := range handledWorkflowIds {
-				if item.Id == workflowId {
-					found = true
-					break
-				}
-			}
-
-			if found {
-				continue
-			}
-
-			handledWorkflowIds = append(handledWorkflowIds, item.Id)
-			err := handleWorkflowAction(request, item)
-			if err != nil {
-				log.Printf("Error in workflow exec: %s", err)
-			}
-		}
-	}
-
-	// FIXME - send the webhookdata to a logging service? Idk
-	//body, err := ioutil.ReadAll(request.Body)
-	//if err != nil {
-	//	log.Println("Failed reading body")
-	//	resp.WriteHeader(401)
-	//	resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
-	//	return
-	//}
-
-	//callback, err := http.Post(callbackUrl, "application/json", bytes.NewBuffer(body))
-	//if err != nil {
-	//	log.Printf("Failed sending callback to %s", callbackUrl)
-	//}
-
-	resp.WriteHeader(200)
-	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
-	return
-}
-
-func loadConfiguration(fullUrl string, apikey string) error {
-	client := &http.Client{}
-
-	req, err := http.NewRequest(
-		"GET",
-		fullUrl,
-		nil,
-	)
-
-	if err != nil {
-		log.Printf("Error making http request: %s", req)
-		return err
-	}
-
-	req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
-	req.Header.Add("Content-Type", "application/json")
-	resp, err := client.Do(req)
-	if err != nil {
-		log.Printf("Error in http request: %s", req)
-		return err
-	}
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		log.Printf("Error reading response: %s", req)
-		return err
-	}
-
-	err = json.Unmarshal(body, &hook)
-	if err != nil {
-		log.Printf("Failed unmarshaling hook API", req)
-		return err
-	}
-
-	log.Printf("%#v", hook)
-	log.Println(hook.Actions)
-	return nil
-}
-
-func webhook() {
-	// FIXME - remove static
-	ip := "0.0.0.0"
-
-	// Basic webserver stuff
-	baseFilePath := os.Getenv("URIPATH")
-	basePort := os.Getenv("HOOKPORT")
-	callbackUrl := os.Getenv("CALLBACKURL")
-	apikey := os.Getenv("APIKEY")
-	hookId := os.Getenv("HOOKID")
-
-	if len(baseFilePath) == 0 {
-		log.Println("Env URIPATH not set")
-		os.Exit(3)
-	}
-
-	if len(basePort) == 0 {
-		log.Println("Env HOOKPORT not set")
-		os.Exit(3)
-	}
-
-	if len(callbackUrl) == 0 {
-		log.Println("Env CALLBACKURL not set")
-		os.Exit(3)
-	}
-
-	if len(apikey) == 0 {
-		log.Println("Env APIKEY not set")
-		os.Exit(3)
-	}
-
-	if len(hookId) == 0 {
-		log.Println("Env HOOKID not set")
-		os.Exit(3)
-	}
-
-	log.Println("Loading hook configuration")
-	err := loadConfiguration(
-		fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId),
-		apikey,
-	)
-
-	if err != nil {
-		log.Fatalf("Error loading config: %s", err)
-	}
-
-	// Optional
-	// if len(callbackOpts) == 0 {
-	// 	log.Println("Env CALLBACKOPTS not set")
-	// 	os.Exit(3)
-	// }
-
-	port := fmt.Sprintf(":%s", basePort)
-	log.Printf("Starting webhook on %s%s with path %s", ip, port, baseFilePath)
-
-	// Routing
-	mux := mux.NewRouter()
-	mux.SkipClean(true)
-
-	// FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend
-	mux.HandleFunc(baseFilePath, callback).Methods("POST")
-
-	handlers.LoggingHandler(os.Stdout, mux)
-	loggedRouter := handlers.LoggingHandler(os.Stdout, mux)
-
-	err = http.ListenAndServe(
-		port,
-		loggedRouter,
-	)
-
-	if err != nil {
-		log.Fatal("ListenAndServer: ", err)
-	}
-}
-
-func F(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
-	w.Write([]byte(r.Header.Get("X-Forwarded-For")))
-}
-
-func main() {
-	webhook()
-}

frontend/src/App.js

@@ -38,10 +38,10 @@ import AlertTemplate from "react-alert-template-basic";
 import { positions, Provider } from "react-alert";
 
 // Testing - localhost
-//const globalUrl = "http://192.168.3.6:5001"
+const globalUrl = "http://192.168.3.6:5001"
 
 // Production - backend proxy forwarding in nginx
-const globalUrl = window.location.origin
+//const globalUrl = window.location.origin
 
 const surfaceColor = "#27292D"
 const inputColor = "#383B40"
@@ -77,7 +77,7 @@ const App = (message, props) => {
   }})
 
 	if (isLoaded && !isLoggedIn && (!window.location.pathname.startsWith("/login") && (!window.location.pathname.startsWith("/docs") && (!window.location.pathname.startsWith("/adminsetup"))))) {
-		window.location = "login"
+		window.location = "/login"
 	}
 
   const checkLogin = () => {

frontend/src/Apps.js

@@ -286,7 +286,7 @@ const Apps = (props) => {
 		var description = selectedApp.description
 
 		const url = "/apps/edit/"+selectedApp.id
-		var editButton = selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ?
+		var editButton = selectedApp.activated && selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated ?
 			<Link to={url} style={{textDecoration: "none"}}>
 				<Button
 					variant="outlined"
@@ -297,6 +297,17 @@ const Apps = (props) => {
 					Edit app	
 				</Button></Link> : null
 
+		var activateButton = selectedApp.generated && !selectedApp.activated ?
+			<Link to={url} style={{textDecoration: "none"}}>
+				<Button
+					variant="outlined"
+					component="label"
+					color="primary"
+					style={{marginTop: "10px"}}
+				>
+					Activate App	
+				</Button></Link> : null
+
 
 		var deleteButton = (selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true) ?
 				<Button
@@ -377,6 +388,7 @@ const Apps = (props) => {
 					</div>
 				: null}
 
+				{activateButton}
 				{editButton}
 				{deleteButton}
 			</div>
@@ -384,11 +396,12 @@ const Apps = (props) => {
 			null
 
 		return(
-			<div>
+			<div style={{}}>
 				<Paper square style={uploadViewPaperStyle}>
 					<div style={{width: "100%", margin: 25}}>
 						<h2>App Creator</h2>
-						<a href="https://github.com/frikky/OpenAPI-security-definitions" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
+						<a href="/docs/apps" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">How it works</a>
+						&nbsp;- <a href="https://github.com/frikky/OpenAPI-security-definitions" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
 						&nbsp;- <a href="https://apis.guru/browse-apis/" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">OpenAPI directory</a>
 						<div/>
 						Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available.