marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
shuffle-cracked/backend/go-app/main.go
T
Frikky f47147d794 Fixed go mod issues with new runner handler
2026-05-07 13:50:22 +02:00

6162 lines
193 KiB
Go
Executable File
Raw Permalink Blame History

package main
import (
uuid "github.com/satori/go.uuid"
"github.com/shuffle/shuffle-shared"
singul "github.com/shuffle/singul/pkg"
//"net/http/pprof"
"archive/zip"
"bufio"
"bytes"
"context"
"crypto/md5"
"strconv"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"os/exec"
"net/http/httptest"
"strings"
"time"
"github.com/frikky/kin-openapi/openapi2"
"github.com/frikky/kin-openapi/openapi2conv"
"github.com/frikky/kin-openapi/openapi3"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
http2 "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
// Random
"sort"
xj "github.com/basgys/goxml2json"
newscheduler "github.com/carlescere/scheduler"
"golang.org/x/crypto/bcrypt"
"gopkg.in/yaml.v3"
// Web
"github.com/gorilla/mux"
//http2 "gopkg.in/src-d/go-git.v5/plumbing/transport/http"
//http2 "github.com/go-git/go-git/plumbing/transport/http"
)
// This is used to handle onprem vs offprem databases etc
var gceProject = "shuffle"
var bucketName = "shuffler.appspot.com"
var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps"
var baseDockerName = "frikky/shuffle"
var registryName = "registry.hub.docker.com"
var runningEnvironment = "onprem"
var syncUrl = "https://shuffler.io"
var debug = false
//var syncUrl = "http://localhost:5002"
type retStruct struct {
Success bool `json:"success"`
SyncFeatures shuffle.SyncFeatures `json:"sync_features"`
SessionKey string `json:"session_key"`
IntervalSeconds int64 `json:"interval_seconds"`
Reason string `json:"reason"`
Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"`
Licensed bool `json:"licensed"`
CloudSyncUrl string `json:"cloud_sync_url,omitempty"`
}
type Contact struct {
Firstname string `json:"firstname"`
Lastname string `json:"lastname"`
Title string `json:"title"`
Companyname string `json:"companyname"`
Phone string `json:"phone"`
Email string `json:"email"`
Message string `json:"message"`
}
type Translator struct {
Src struct {
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value,noindex"`
Description string `json:"description" datastore:"description,noindex"`
Required string `json:"required" datastore:"required"`
Type string `json:"type" datastore:"type"`
Schema struct {
Type string `json:"type" datastore:"type"`
} `json:"schema" datastore:"schema"`
} `json:"src" datastore:"src"`
Dst struct {
Name string `json:"name" datastore:"name"`
Value string `json:"value" datastore:"value,noindex"`
Type string `json:"type" datastore:"type"`
Description string `json:"description" datastore:"description,noindex"`
Required string `json:"required" datastore:"required"`
Schema struct {
Type string `json:"type" datastore:"type"`
} `json:"schema" datastore:"schema"`
} `json:"dst" datastore:"dst"`
}
type Appconfig struct {
Key string `json:"key" datastore:"key"`
Value string `json:"value" datastore:"value,noindex"`
}
type ScheduleApp struct {
Foldername string `json:"foldername" datastore:"foldername,noindex"`
Name string `json:"name" datastore:"name,noindex"`
Id string `json:"id" datastore:"id,noindex"`
Description string `json:"description" datastore:"description,noindex"`
Action string `json:"action" datastore:"action,noindex"`
Config []Appconfig `json:"config,omitempty" datastore:"config,noindex"`
}
type AppInfo struct {
SourceApp ScheduleApp `json:"sourceapp,omitempty" datastore:"sourceapp,noindex"`
DestinationApp ScheduleApp `json:"destinationapp,omitempty" datastore:"destinationapp,noindex"`
}
// May 2020: Reused for onprem schedules - Id, Seconds, WorkflowId and argument
type ScheduleOld struct {
Id string `json:"id" datastore:"id"`
StartNode string `json:"start_node" datastore:"start_node"`
Seconds int `json:"seconds" datastore:"seconds"`
WorkflowId string `json:"workflow_id" datastore:"workflow_id", `
Argument string `json:"argument" datastore:"argument"`
WrappedArgument string `json:"wrapped_argument" datastore:"wrapped_argument"`
AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"`
Finished bool `json:"finished" finished:"id"`
BaseAppLocation string `json:"base_app_location" datastore:"baseapplocation,noindex"`
Translator []Translator `json:"translator,omitempty" datastore:"translator"`
Org string `json:"org" datastore:"org"`
CreatedBy string `json:"createdby" datastore:"createdby"`
Availability string `json:"availability" datastore:"availability"`
CreationTime int64 `json:"creationtime" datastore:"creationtime,noindex"`
LastModificationtime int64 `json:"lastmodificationtime" datastore:"lastmodificationtime,noindex"`
LastRuntime int64 `json:"lastruntime" datastore:"lastruntime,noindex"`
Frequency string `json:"frequency" datastore:"frequency,noindex"`
Environment string `json:"environment" datastore:"environment"`
}
// Returned from /GET /schedules
type Schedules struct {
Schedules []ScheduleOld `json:"schedules"`
Success bool `json:"success"`
}
type ScheduleApps struct {
Apps []ApiYaml `json:"apps"`
Success bool `json:"success"`
}
// The yaml that is uploaded
type ApiYaml struct {
Name string `json:"name" yaml:"name" required:"true datastore:"name"`
Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"`
Id string `json:"id" yaml:"id",required:"true, datastore:"id"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"`
ContactInfo struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Url string `json:"url" datastore:"url" yaml:"url"`
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info"`
Types []string `json:"types" datastore:"types" yaml:"types"`
Input []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
InputParameters []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Required string `json:"required" datastore:"required" yaml:"required"`
Schema struct {
Type string `json:"type" datastore:"type" yaml:"type"`
} `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"`
OutputParameters []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Required string `json:"required" datastore:"required" yaml:"required"`
Schema struct {
Type string `json:"type" datastore:"type" yaml:"type"`
} `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"`
Config []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Required string `json:"required" datastore:"required" yaml:"required"`
Schema struct {
Type string `json:"type" datastore:"type" yaml:"type"`
} `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"config" datastore:"config" yaml:"config"`
} `json:"input" datastore:"input" yaml:"input"`
Output []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Config []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Required string `json:"required" datastore:"required" yaml:"required"`
Schema struct {
Type string `json:"type" datastore:"type" yaml:"type"`
} `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"config" datastore:"config" yaml:"config"`
InputParameters []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Required string `json:"required" datastore:"required" yaml:"required"`
Schema struct {
Type string `json:"type" datastore:"type" yaml:"type"`
} `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"`
OutputParameters []struct {
Name string `json:"name" datastore:"name" yaml:"name"`
Description string `json:"description" datastore:"description,noindex" yaml:"description"`
Required string `json:"required" datastore:"required" yaml:"required"`
Schema struct {
Type string `json:"type" datastore:"type" yaml:"type"`
} `json:"schema" datastore:"schema" yaml:"schema"`
} `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"`
} `json:"output" datastore:"output" yaml:"output"`
}
type Hooks struct {
Hooks []Hook `json:"hooks"`
Success bool `json:"-"`
}
type Info struct {
Url string `json:"url" datastore:"url"`
Name string `json:"name" datastore:"name"`
Description string `json:"description" datastore:"description,noindex"`
}
// Actions to be done by webhooks etc
// Field is the actual field to use from json
type HookAction struct {
Type string `json:"type" datastore:"type"`
Name string `json:"name" datastore:"name"`
Id string `json:"id" datastore:"id"`
Field string `json:"field" datastore:"field"`
}
type Hook struct {
Id string `json:"id" datastore:"id"`
Start string `json:"start" datastore:"start"`
Info Info `json:"info" datastore:"info"`
Actions []HookAction `json:"actions" datastore:"actions,noindex"`
Type string `json:"type" datastore:"type"`
Owner string `json:"owner" datastore:"owner"`
Status string `json:"status" datastore:"status"`
Workflows []string `json:"workflows" datastore:"workflows"`
Running bool `json:"running" datastore:"running"`
OrgId string `json:"org_id" datastore:"org_id"`
Environment string `json:"environment" datastore:"environment"`
}
func GetUsersHandler(w http.ResponseWriter, r *http.Request) {
data := map[string]interface{}{
"id": "12345",
"ts": time.Now().Format(time.RFC3339),
}
b, err := json.Marshal(data)
if err != nil {
http.Error(w, err.Error(), 400)
return
}
w.Write(b)
}
func jsonPrettyPrint(in string) string {
var out bytes.Buffer
err := json.Indent(&out, []byte(in), "", "\t")
if err != nil {
return in
}
return out.String()
}
// Does User exist?
// Does User have permission to view / run this?
// Encoding: /json?
// General authentication
func authenticate(request *http.Request) bool {
authField := "authorization"
authenticationKey := "topkek"
//authFound := false
// This should work right?
for name, headers := range request.Header {
name = strings.ToLower(name)
for _, h := range headers {
if name == authField && h == authenticationKey {
//log.Printf("%v: %v", name, h)
return true
}
}
}
return false
}
func checkError(cmdName string, cmdArgs []string) error {
cmd := exec.Command(cmdName, cmdArgs...)
cmdReader, err := cmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err)
return err
}
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
log.Printf("Out: %s\n", scanner.Text())
}
}()
err = cmd.Start()
if err != nil {
fmt.Fprintln(os.Stderr, "Error starting Cmd", err)
return err
}
err = cmd.Wait()
if err != nil {
fmt.Fprintln(os.Stderr, "Error waiting for Cmd", err)
return err
}
return nil
}
func md5sum(data []byte) string {
hasher := md5.New()
hasher.Write(data)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
return newmd5
}
func md5sumfile(filepath string) string {
dat, err := ioutil.ReadFile(filepath)
if err != nil {
log.Printf("Error in dat: %s", err)
}
hasher := md5.New()
hasher.Write(dat)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
log.Printf("%s: %s", filepath, newmd5)
return newmd5
}
func checkFileExistsLocal(basepath string, filepath string) bool {
User := "test"
// md5sum
// get tmp/results/md5sum/folder/results.json
// parse /tmp/results/md5sum/results.json
path := fmt.Sprintf("%s/%s", basepath, md5sumfile(filepath))
if _, err := os.Stat(path); os.IsNotExist(err) {
//log.Printf("File error for %s: %s", filepath, err)
return false
}
log.Printf("File %s exists. Getting for User %s.", filepath, User)
return true
}
func redirect(w http.ResponseWriter, req *http.Request) {
// remove/add not default ports from req.Host
target := "https://" + req.Host + req.URL.Path
if len(req.URL.RawQuery) > 0 {
target += "?" + req.URL.RawQuery
}
log.Printf("redirect to: %s", target)
http.Redirect(w, req, target,
// see @andreiavrammsd comment: often 307 > 301
http.StatusTemporaryRedirect)
}
// No more emails :)
func checkUsername(Username string) error {
// Stupid first check of email loool
//if !strings.Contains(Username, "@") || !strings.Contains(Username, ".") {
// return errors.New("Invalid Username")
//}
if len(Username) < 3 {
return errors.New("Minimum Username length is 3")
}
return nil
}
func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error {
// Returns false if there is an issue
// Use this for register
err := shuffle.CheckPasswordStrength(username, password)
if err != nil {
log.Printf("[WARNING] Bad password strength: %s", err)
return err
}
err = checkUsername(username)
if err != nil {
log.Printf("[WARNING] Bad Username strength: %s", err)
return err
}
ctx := context.Background()
users, err := shuffle.FindUser(ctx, strings.ToLower(strings.TrimSpace(username)))
if err != nil && len(users) == 0 {
log.Printf("[WARNING] Failed getting user %s: %s", username, err)
return err
}
if len(users) > 0 {
return errors.New(fmt.Sprintf("Username %s already exists", username))
}
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), 8)
if err != nil {
log.Printf("Wrong password for %s: %s", username, err)
return err
}
newUser := new(shuffle.User)
newUser.Username = username
newUser.Password = string(hashedPassword)
newUser.Verified = false
newUser.CreationTime = time.Now().Unix()
newUser.Active = true
newUser.Orgs = []string{org.Id}
if role == "admin" {
newUser.Role = "admin"
newUser.Roles = []string{"admin"}
} else {
newUser.Role = "user"
newUser.Roles = []string{"user"}
}
newUser.ActiveOrg = shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
Role: newUser.Role,
}
if len(apikey) > 0 {
newUser.ApiKey = apikey
}
// set limits
newUser.Limits.DailyApiUsage = 100
newUser.Limits.DailyWorkflowExecutions = 1000
newUser.Limits.DailyCloudExecutions = 100
newUser.Limits.DailyTriggers = 20
newUser.Limits.DailyMailUsage = 100
newUser.Limits.MaxTriggers = 10
newUser.Limits.MaxWorkflows = 10
// Set base info for the user
newUser.Executions.TotalApiUsage = 0
newUser.Executions.TotalWorkflowExecutions = 0
newUser.Executions.TotalAppExecutions = 0
newUser.Executions.TotalCloudExecutions = 0
newUser.Executions.TotalOnpremExecutions = 0
newUser.Executions.DailyApiUsage = 0
newUser.Executions.DailyWorkflowExecutions = 0
newUser.Executions.DailyAppExecutions = 0
newUser.Executions.DailyCloudExecutions = 0
newUser.Executions.DailyOnpremExecutions = 0
verifyToken := uuid.NewV4()
ID := uuid.NewV4()
newUser.Id = ID.String()
newUser.VerificationToken = verifyToken.String()
err = shuffle.SetUser(ctx, newUser, true)
if err != nil {
log.Printf("[ERROR] Problem adding User %s: %s", username, err)
return err
}
neworg, err := shuffle.GetOrg(ctx, org.Id)
if err == nil {
//neworg.Users = append(neworg.Users, *newUser)
for tutorialIndex, tutorial := range neworg.Tutorials {
if tutorial.Name == "Invite teammates" {
neworg.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d users are in your org. Org name and Image change next.", len(neworg.Users))
if len(neworg.Users) > 1 {
neworg.Tutorials[tutorialIndex].Done = true
neworg.Tutorials[tutorialIndex].Link = "/admin?tab=users"
}
break
}
}
err = shuffle.SetOrg(ctx, *neworg, neworg.Id)
if err != nil {
log.Printf("Failed updating org with user %s", newUser.Username)
} else {
log.Printf("[INFO] Successfully updated org with user %s!", newUser.Username)
}
}
return nil
}
func handleRegister(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
// Only admin can CREATE users, but if there are no users, anyone can make (first)
ctx := context.Background()
users, countErr := shuffle.GetAllUsers(ctx)
count := len(users)
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
if (countErr == nil && count > 0) || countErr != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Users already exist. Please go to /login to log into your admin user."}`))
return
}
}
apikey := ""
if count != 0 {
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin (2)"}`))
return
}
} else {
apikey = uuid.NewV4().String()
}
// Gets a struct of Username, password
data, err := shuffle.ParseLoginParameters(resp, request)
if err != nil {
log.Printf("Invalid params: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
role := "user"
if count == 0 {
role = "admin"
}
currentOrg := user.ActiveOrg
if user.ActiveOrg.Id == "" {
log.Printf("[WARNING] There's no active org for the user %s. Checking if there's a single one to assign it to.", user.Username)
orgs, err := shuffle.GetAllOrgs(ctx)
if err == nil && len(orgs) > 0 {
log.Printf("[WARNING] No org exists for user %s. Setting to default (first one)", user.Username)
currentOrg = shuffle.OrgMini{
Id: orgs[0].Id,
Name: orgs[0].Name,
}
} else {
log.Printf("[WARNING] Couldn't find an org to attach to. Create?")
orgSetupName := "default"
orgId := uuid.NewV4().String()
newOrg := shuffle.Org{
Name: orgSetupName,
Id: orgId,
Org: orgSetupName,
Users: []shuffle.User{user},
Roles: []string{"admin", "user"},
CloudSync: false,
}
err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed setting init organization: %s", err)
} else {
log.Printf("[DEBUG] Successfully created the default org!")
defaultEnv := os.Getenv("ENVIRONMENT_NAME")
if len(defaultEnv) == 0 {
defaultEnv = "Shuffle"
log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv)
}
item := shuffle.Environment{
Name: defaultEnv,
Type: "onprem",
OrgId: orgId,
Default: true,
Id: uuid.NewV4().String(),
}
err = shuffle.SetEnvironment(ctx, &item)
if err != nil {
log.Printf("[WARNING] Failed setting up new environment for new org: %s", err)
}
currentOrg = shuffle.OrgMini{
Id: newOrg.Id,
Name: newOrg.Name,
}
user.ActiveOrg = currentOrg
}
}
}
err = createNewUser(data.Username, data.Password, role, apikey, currentOrg)
if err != nil {
if strings.Contains(err.Error(), "already exists") {
// Assign it to the org
log.Printf("[WARNING] User %s already exists. Assigning to org %s", data.Username, currentOrg.Name)
// Get the user
users, err := shuffle.FindUser(ctx, data.Username)
if err != nil || len(users) == 0 {
log.Printf("[WARNING] Failed finding user %s: %s", data.Username, err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
newUser := users[0]
if !shuffle.ArrayContains(newUser.Orgs, currentOrg.Id) {
newUser.Orgs = append(newUser.Orgs, currentOrg.Id)
}
if newUser.ActiveOrg.Id == "" || newUser.ActiveOrg.Name == "" {
newUser.ActiveOrg = currentOrg
}
err = shuffle.SetUser(ctx, &newUser, true)
if err != nil {
log.Printf("[WARNING] Failed updating the user %s: %s", data.Username, err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
log.Printf("[INFO] %s Successfully re-added to org %s (%s)", data.Username, currentOrg.Name, currentOrg.Id)
return
} else {
log.Printf("[WARNING] Failed registering user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "apikey": "%s"}`, apikey)))
log.Printf("[INFO] %s Successfully registered.", data.Username)
}
func handleCookie(request *http.Request) bool {
c, err := request.Cookie("session_token")
if err != nil {
return false
}
if len(c.Value) == 0 {
return false
}
return true
}
// Returns whether the user is logged in or not etc.
// Also has more data about the user and org
func handleInfo(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
userInfo, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in handleInfo: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := context.Background()
// This is a long check to see if an inactive admin can access the site
parsedAdmin := "false"
if userInfo.Role == "admin" {
parsedAdmin = "true"
}
if !userInfo.Active {
if userInfo.Role == "admin" {
parsedAdmin = "true"
ctx := context.Background()
users, err := shuffle.GetAllUsers(ctx)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to get other users when verifying admin user"}`))
return
}
activeFound := false
adminFound := false
for _, user := range users {
if user.Id == userInfo.Id {
continue
}
if user.Role != "admin" {
continue
}
if user.Active {
activeFound = true
}
adminFound = true
}
// Must ALWAYS be an active admin
// Will return no access if another admin is active
if !adminFound {
log.Printf("NO OTHER ADMINS FOUND - CONTINUE!")
} else {
//
if activeFound {
log.Printf("OTHER ACTIVE ADMINS FOUND - CAN'T PASS")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This user is locked"}`))
return
} else {
log.Printf("NO OTHER ADMINS FOUND - CONTINUE!")
}
}
} else {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This user is locked"}`))
return
}
}
go shuffle.CheckSessionOrgs(ctx, userInfo)
//log.Printf("%s %s", session.Session, UserInfo.Session)
//if session.Session != userInfo.Session {
// log.Printf("Session %s is not the same as %s for %s. %s", userInfo.Session, session.Session, userInfo.Username, err)
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false, "reason": ""}`))
// return
//}
expiration := time.Now().Add(3600 * time.Second)
sessionCookie := shuffle.ConstructSessionCookie(userInfo.Session, expiration)
http.SetCookie(resp, sessionCookie)
// Updating user info if there's something wrong
if len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0 {
if len(userInfo.Orgs) == 0 || (len(userInfo.Orgs) > 0 && userInfo.Orgs[0] == "") {
orgs, err := shuffle.GetAllOrgs(ctx)
log.Printf("[INFO] Fixing organization for user %s (%s). Found orgs: %d", userInfo.Username, userInfo.Id, len(orgs))
if err == nil && len(orgs) > 0 {
for _, org := range orgs {
if len(org.Id) == 0 {
continue
}
// Prolly some way here to jump into another org
// when you have access to the DB
userInfo.ActiveOrg = shuffle.OrgMini{
Name: org.Name,
Id: org.Id,
Role: "admin",
}
userInfo.Orgs = []string{org.Id}
break
}
}
// Make a new one in case we couldn't find one
if len(userInfo.ActiveOrg.Id) == 0 {
orgSetupName := "default"
orgId := uuid.NewV4().String()
newOrg := shuffle.Org{
Name: orgSetupName,
Id: orgId,
Org: orgSetupName,
Users: []shuffle.User{},
Roles: []string{"admin", "user"},
CloudSync: false,
}
err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
if err == nil {
userInfo.ActiveOrg = shuffle.OrgMini{
Name: newOrg.Name,
Id: newOrg.Id,
Role: "admin",
}
userInfo.Orgs = []string{newOrg.Id}
} else {
log.Printf("[WARNING] Failed to set new org: %s", err)
}
}
// Set user
err = shuffle.SetUser(ctx, &userInfo, true)
if err != nil {
log.Printf("[WARNING] Failed fixing org info for user %s (%s)", userInfo.Username, userInfo.Id)
} else {
log.Printf("[INFO] Set organization for %s (%s) to be %s (%s)", userInfo.Username, userInfo.Id, userInfo.ActiveOrg.Name, userInfo.ActiveOrg.Id)
}
} else if len(userInfo.Orgs) > 0 && userInfo.Orgs[0] != "" {
_, err := shuffle.GetOrg(ctx, userInfo.Orgs[0])
if err != nil {
orgs, err := shuffle.GetAllOrgs(ctx)
if err == nil {
newStringOrgs := []string{}
newOrgs := []shuffle.Org{}
for _, org := range orgs {
if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) {
newOrgs = append(newOrgs, org)
newStringOrgs = append(newStringOrgs, org.Id)
}
}
if len(newOrgs) > 0 {
userInfo.ActiveOrg = shuffle.OrgMini{
Id: newOrgs[0].Id,
Name: newOrgs[0].Name,
}
userInfo.Orgs = newStringOrgs
err = shuffle.SetUser(ctx, &userInfo, true)
if err != nil {
log.Printf("Error patching User for activeOrg: %s", err)
} else {
log.Printf("Updated the users' org")
}
}
} else {
log.Printf("Failed getting orgs for user. Major issue.: %s", err)
}
} else {
// 1. Check if the org exists by ID
// 2. if it does, overwrite user
userInfo.ActiveOrg = shuffle.OrgMini{
Id: userInfo.Orgs[0],
}
err = shuffle.SetUser(ctx, &userInfo, true)
if err != nil {
log.Printf("[INFO] Error patching User for activeOrg: %s", err)
}
}
}
}
org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id)
if err != nil {
log.Printf("[DEBUG] Failed to get org during getinfo: %s", err)
}
childOrgs := []shuffle.Org{}
if len(org.CreatorOrg) > 0 {
childOrgs, _, err = shuffle.GetAllChildOrgs(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[ERROR] Failed to get child orgs during getinfo: %s", err)
childOrgs = []shuffle.Org{}
}
}
failToLoadOrgs := []string{}
sort.Slice(childOrgs, func(i, j int) bool {
return childOrgs[i].Created < childOrgs[j].Created
})
parentOrg := &shuffle.Org{}
if len(org.CreatorOrg) > 0 {
parentOrg, err = shuffle.GetOrg(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[ERROR] Failed to get parent org during getinfo: %s", err)
parentOrg = &shuffle.Org{}
}
licenseOrg := shuffle.HandleCheckLicense(ctx, *parentOrg)
parentOrg = &licenseOrg
}
limit := 3
if parentOrg.Licensed && parentOrg.SyncFeatures.MultiTenant.Active {
limit = int(parentOrg.SyncFeatures.MultiTenant.Limit)
}
for index, org := range childOrgs {
if index < limit {
continue
}
failToLoadOrgs = append(failToLoadOrgs, org.Id)
}
if len(org.CreatorOrg) > 0 && len(childOrgs) > limit && shuffle.ArrayContains(failToLoadOrgs, userInfo.ActiveOrg.Id) {
userInfo.ActiveOrg = shuffle.OrgMini{
Id: parentOrg.Id,
Name: parentOrg.Name,
Role: userInfo.Role,
Branding: parentOrg.Branding,
Image: parentOrg.Image,
}
if parentOrg.Licensed {
log.Printf("[INFO] Parent org %s is licensed. But Multi-tenant feature is not active. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username)
} else {
log.Printf("[INFO] Parent org %s has more than 3 child orgs and is not licensed. Moving user %s to parent org %s", parentOrg.Name, userInfo.Username)
}
err = shuffle.SetUser(ctx, &userInfo, false)
if err != nil {
log.Printf("[WARNING] Failed setting user to parent org: %s", err)
}
reason := "Parent org has more than 3 child orgs and is not licensed. Moving user to parent org. Contact support@shuffler.io for more information"
if parentOrg.Licensed {
reason = fmt.Sprintf("Parent organization is licensed, but the maximum number of sub-organizations (%d) has been reached. You have been moved to the parent organization. Please contact support@shuffler.io for further assistance.", parentOrg.SyncFeatures.MultiTenant.Limit)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "` + reason + `", "switch_parent": true}`))
return
}
//if err == nil {
if len(org.Id) > 0 {
if userInfo.Role == "" {
//err = shuffle.SetUser(ctx, &userInfo, false)
for _, user := range org.Users {
if user.Id != userInfo.Id {
continue
}
userInfo.ActiveOrg.Role = user.Role
}
}
userInfo.ActiveOrg = shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
CreatorOrg: org.CreatorOrg,
ChildOrgs: org.ChildOrgs,
Role: userInfo.ActiveOrg.Role,
Image: org.Image,
}
if parsedAdmin == "false" {
// Validating admin user again just to make sure
// This is to avoid issues for the first org ever
for _, user := range org.Users {
if user.Id != userInfo.Id {
continue
}
if user.Role == "admin" {
break
}
}
}
}
orgPriorities := org.Priorities
if len(org.Priorities) < 10 {
//log.Printf("[WARNING] Should find and add priorities as length is less than 10 for org %s", userInfo.ActiveOrg.Id)
newPriorities, err := shuffle.GetPriorities(ctx, userInfo, org)
if err != nil {
log.Printf("[WARNING] Failed getting new priorities for org %s: %s", org.Id, err)
//orgPriorities = []shuffle.Priority{}
} else {
orgPriorities = newPriorities
// A way to manage them over time
}
}
orgInterests := org.Interests
userInfo.ActiveOrg.Users = []shuffle.UserMini{}
userOrgs := []shuffle.OrgMini{}
for _, item := range userInfo.Orgs {
if item == userInfo.ActiveOrg.Id {
userOrgs = append(userOrgs, userInfo.ActiveOrg)
continue
}
org, err := shuffle.GetOrg(ctx, item)
if len(org.Id) > 0 {
userOrgs = append(userOrgs, shuffle.OrgMini{
Id: org.Id,
Name: org.Name,
CreatorOrg: org.CreatorOrg,
Image: org.Image,
})
} else {
log.Printf("[WARNING] Failed to get org %s (%s) for user %s. Error: %#v", org.Name, item, userInfo.Username, err)
}
}
// FIXME: This is bad, but we've had a lot of bugs with edit users, and this is the quick fix.
if userInfo.Role == "" && userInfo.ActiveOrg.Role == "" && parsedAdmin == "false" {
userInfo.Role = "admin"
userInfo.ActiveOrg.Role = "admin"
parsedAdmin = "true"
err = shuffle.SetUser(ctx, &userInfo, true)
if err != nil {
log.Printf("[WARNING] Automatically asigning user as admin to their org because they don't have a role at all failed: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
} else {
log.Printf("[DEBUG] Made user %s org-admin as they didn't have any role specified", err)
}
}
if parsedAdmin == "true" {
userInfo.Role = "admin"
userInfo.ActiveOrg.Role = "admin"
}
chatDisabled := false
if os.Getenv("SHUFFLE_CHAT_DISABLED") == "true" {
chatDisabled = true
}
userOrgs = shuffle.SortOrgList(userOrgs)
tutorialsFinished := []shuffle.Tutorial{}
for _, tutorial := range userInfo.PersonalInfo.Tutorials {
tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{
Name: tutorial,
})
}
if len(org.SecurityFramework.SIEM.Name) > 0 || len(org.SecurityFramework.Network.Name) > 0 || len(org.SecurityFramework.EDR.Name) > 0 || len(org.SecurityFramework.Cases.Name) > 0 || len(org.SecurityFramework.IAM.Name) > 0 || len(org.SecurityFramework.Assets.Name) > 0 || len(org.SecurityFramework.Intel.Name) > 0 || len(org.SecurityFramework.Communication.Name) > 0 {
tutorialsFinished = append(tutorialsFinished, shuffle.Tutorial{
Name: "find_integrations",
})
}
for _, tutorial := range org.Tutorials {
tutorialsFinished = append(tutorialsFinished, tutorial)
}
licensed := shuffle.IsLicensed(ctx, *org)
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
if err != nil {
log.Printf("{WARNING] Failed getting apps (getworkflowapps): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
orgApps := workflowapps
activatedAppIds := []string{}
for _, app := range orgApps {
activatedAppIds = append(activatedAppIds, app.ID)
}
parsedStatus := []string{}
if len(org.ManagerOrgs) > 0 || userInfo.ActiveOrg.CreatorOrg != "" {
parsedStatus = append(parsedStatus, "sub_org")
parentOrgId := userInfo.ActiveOrg.CreatorOrg
if len(userInfo.ActiveOrg.CreatorOrg) == 0 {
parentOrgId = org.ManagerOrgs[0].Id
}
// Check for licensing/branding of parent and override
parentOrg, err := shuffle.GetOrg(ctx, parentOrgId)
if err == nil {
parent := shuffle.HandleCheckLicense(ctx, *parentOrg)
if parent.SyncFeatures.Branding.Active {
parsedStatus = append(parsedStatus, "integration_partner")
// except theme take from parent org
org.Branding.EnableChat = parentOrg.Branding.EnableChat
org.Branding.HomeUrl = parentOrg.Branding.HomeUrl
org.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference
org.Branding.SupportEmail = parentOrg.Branding.SupportEmail
org.Branding.LogoutUrl = parentOrg.Branding.LogoutUrl
org.Branding.BrandColor = parentOrg.Branding.BrandColor
org.Branding.BrandName = parentOrg.Branding.BrandName
org.Branding.GlobalUser = parentOrg.Branding.GlobalUser
if len(org.Branding.Theme) == 0 {
org.Branding.Theme = parentOrg.Branding.Theme
}
userInfo.ActiveOrg.Branding = parentOrg.Branding
userInfo.ActiveOrg.Image = parentOrg.Image
userInfo.ActiveOrg.Branding.DocumentationLink = parentOrg.Defaults.DocumentationReference
userInfo.ActiveOrg.Branding.BrandColor = parentOrg.Branding.BrandColor
userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail
userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl
userInfo.ActiveOrg.Branding.BrandName = parentOrg.Branding.BrandName
if len(org.Branding.Theme) == 0 {
userInfo.ActiveOrg.Branding.Theme = parentOrg.Branding.Theme
} else {
userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme
}
// check whether current is global user or not? means is user part of parent org or not
for _, user := range parentOrg.Users {
if user.Id == userInfo.Id && user.Role == "admin" {
userInfo.ActiveOrg.Branding.GlobalUser = true
break
}
}
} else {
userInfo.ActiveOrg.Branding = shuffle.OrgBranding{}
}
}
} else {
// for parent org branding
licenseOrg := shuffle.HandleCheckLicense(ctx, *org)
org = &licenseOrg
if org.SyncFeatures.Branding.Active {
userInfo.ActiveOrg.Branding.Theme = org.Branding.Theme
userInfo.ActiveOrg.Branding.DocumentationLink = org.Defaults.DocumentationReference
userInfo.ActiveOrg.Branding.SupportEmail = org.Branding.SupportEmail
userInfo.ActiveOrg.Branding.LogoutUrl = org.Branding.LogoutUrl
userInfo.ActiveOrg.Branding.BrandColor = org.Branding.BrandColor
userInfo.ActiveOrg.Branding.BrandName = org.Branding.BrandName
parsedStatus = append(parsedStatus, "integration_partner")
} else {
userInfo.ActiveOrg.Branding = shuffle.OrgBranding{}
}
}
aiEnabled := os.Getenv("OPENAI_API_URL") != "" && os.Getenv("AI_MODEL") != ""
returnValue := shuffle.HandleInfo{
Success: true,
Username: userInfo.Username,
Admin: parsedAdmin,
Id: userInfo.Id,
Orgs: userOrgs,
ActiveOrg: userInfo.ActiveOrg,
Cookies: []shuffle.SessionCookie{
shuffle.SessionCookie{
Key: "session_token",
Value: userInfo.Session,
Expiration: expiration.Unix(),
},
},
EthInfo: userInfo.EthInfo,
ChatDisabled: chatDisabled,
Tutorials: tutorialsFinished,
Interests: orgInterests,
Priorities: orgPriorities,
Licensed: licensed,
ActiveApps: activatedAppIds,
Theme: userInfo.Theme,
OrgStatus: parsedStatus,
AIEnabled: aiEnabled,
}
returnData, err := json.Marshal(returnValue)
if err != nil {
log.Printf("[WARNING] Failed marshalling info in handleinfo: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(returnData))
}
type passwordReset struct {
Password1 string `json:"newpassword"`
Password2 string `json:"newpassword2"`
Reference string `json:"reference"`
}
func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
ctx := context.Background()
users, err := shuffle.GetAllUsers(ctx)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
count := len(users)
if count == 0 {
log.Printf("[WARNING] No users - redirecting for management user")
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`)))
return
}
baseSSOUrl := ""
handled := []string{}
for _, user := range users {
if shuffle.ArrayContains(handled, user.ActiveOrg.Id) {
continue
}
handled = append(handled, user.ActiveOrg.Id)
org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Error getting org in admin check: %s", err)
continue
}
// No childorg setup, only parent org
// if len(org.ManagerOrgs) > 0 || len(org.CreatorOrg) > 0 {
// continue
// }
// Should run calculations
if len(org.SSOConfig.OpenIdAuthorization) > 0 {
baseSSOUrl = shuffle.GetOpenIdUrl(request, *org)
if err != nil {
log.Printf("[ERROR] Failed getting OpenID URL for org %s: %s", org.Name, err)
}
break
}
if len(org.SSOConfig.SSOEntrypoint) > 0 {
log.Printf("[DEBUG] Found SAML SSO url: %s", org.SSOConfig.SSOEntrypoint)
baseSSOUrl = org.SSOConfig.SSOEntrypoint
break
}
}
//log.Printf("[DEBUG] OpenID URL: %s", baseSSOUrl)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, baseSSOUrl)))
}
func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org {
//found := false
//for _, id := range user.Orgs {
// if user.ActiveOrg.Id == id {
// found = true
// break
// }
//}
//if !found {
// user.Orgs = append(user.Orgs, user.ActiveOrg.Id)
//}
//// Might be vulnerable to timing attacks.
//for _, orgId := range user.Orgs {
// if len(orgId) == 0 {
// continue
// }
// org, err := shuffle.GetOrg(ctx, orgId)
// if err != nil {
// log.Printf("Error getting org %s", orgId)
// continue
// }
// orgIndex := 0
// userFound := false
// for index, orgUser := range org.Users {
// if orgUser.Id == user.Id {
// orgIndex = index
// userFound = true
// break
// }
// }
// if userFound {
// user.PrivateApps = []WorkflowApp{}
// user.Executions = ExecutionInfo{}
// user.Limits = UserLimits{}
// user.Authentication = []UserAuth{}
// org.Users[orgIndex] = *user
// } else {
// org.Users = append(org.Users, *user)
// }
// err = shuffle.SetOrg(ctx, *org, orgId)
// if err != nil {
// log.Printf("Failed setting org %s", orgId)
// }
//}
return org
}
func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User {
found := false
for _, id := range user.Orgs {
if user.ActiveOrg.Id == id {
found = true
break
}
}
if !found {
user.Orgs = append(user.Orgs, user.ActiveOrg.Id)
}
// Might be vulnerable to timing attacks.
for _, orgId := range user.Orgs {
if len(orgId) == 0 {
continue
}
org, err := shuffle.GetOrg(ctx, orgId)
if err != nil {
log.Printf("Error getting org %s", orgId)
continue
}
orgIndex := 0
userFound := false
for index, orgUser := range org.Users {
if orgUser.Id == user.Id {
orgIndex = index
userFound = true
break
}
}
if userFound {
user.PrivateApps = []shuffle.WorkflowApp{}
user.Executions = shuffle.ExecutionInfo{}
user.Limits = shuffle.UserLimits{}
user.Authentication = []shuffle.UserAuth{}
org.Users[orgIndex] = *user
} else {
org.Users = append(org.Users, *user)
}
err = shuffle.SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("Failed setting org %s", orgId)
}
}
return user
}
// Used for testing only. Shouldn't impact production.
/*
func shuffle.HandleCors(resp http.ResponseWriter, request *http.Request) bool {
// Used for Codespace dev
allowedOrigins := "https://frikky-shuffle-5gvr4xx62w64-3000.githubpreview.dev"
//origin := request.Header["Origin"]
//log.Printf("Origin: %s", origin)
//allowedOrigins := "http://localhost:3002"
resp.Header().Set("Vary", "Origin")
resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me, Authorization")
resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, PATCH")
resp.Header().Set("Access-Control-Allow-Credentials", "true")
resp.Header().Set("Access-Control-Allow-Origin", allowedOrigins)
if request.Method == "OPTIONS" {
resp.WriteHeader(200)
resp.Write([]byte("OK"))
return true
}
return false
}
*/
func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (map[string]interface{}, error) {
body, err := ioutil.ReadAll(request.Body)
if err != nil {
return nil, err
}
log.Printf("Parsing data: %s", string(body))
var t map[string]interface{}
err = json.Unmarshal(body, &t)
if err == nil {
log.Printf("PARSED!! :)")
return t, nil
}
// Translate XML to json in case of an XML blob.
// FIXME - use Content-Type and Accept headers
xml := strings.NewReader(string(body))
curjson, err := xj.Convert(xml)
if err != nil {
return t, err
}
//log.Println(curjson.String())
//log.Printf("Parsing json a second time: %s", string(curjson.String()))
err = json.Unmarshal(curjson.Bytes(), &t)
if err != nil {
return t, nil
}
envelope := t["Envelope"].(map[string]interface{})
curbody := envelope["Body"].(map[string]interface{})
//log.Println(curbody)
// ALWAYS handle strings only
// FIXME - remove this and get it from config or something
requiredField := "symptomDescription"
_, found := SearchNested(curbody, requiredField)
// Maxdepth
maxiter := 5
// Need to look for parent of the item, as that is most likely root
if found {
cnt := 0
var previousDifferentItem map[string]interface{}
var previousItem map[string]interface{}
_ = previousItem
for {
if cnt == maxiter {
break
}
// Already know it exists
key, realItem, _ := SearchNestedParent(curbody, requiredField)
// First should ALWAYS work since we already have recursion checked
if len(previousDifferentItem) == 0 {
previousDifferentItem = realItem.(map[string]interface{})
}
switch t := realItem.(type) {
case map[string]interface{}:
previousItem = realItem.(map[string]interface{})
curbody = realItem.(map[string]interface{})
default:
// Gets here if it's not an object
_ = t
//log.Printf("hi %#v", previousItem)
return previousItem, nil
}
_ = key
cnt += 1
}
}
//key, realItem, found = SearchNestedParent(newbody, requiredField)
//if !found {
// log.Println("NOT FOUND!")
//}
////log.Println(realItem[requiredField].(map[string]interface{}))
//log.Println(realItem[requiredField])
//log.Printf("FOUND PARENT :): %s", key)
return t, nil
}
// SearchNested searches a nested structure consisting of map[string]interface{}
// and []interface{} looking for a map with a specific key name.
// If found SearchNested returns the value associated with that key, true
func SearchNestedParent(obj interface{}, key string) (string, interface{}, bool) {
switch t := obj.(type) {
case map[string]interface{}:
if v, ok := t[key]; ok {
return "", v, ok
}
for k, v := range t {
if _, ok := SearchNested(v, key); ok {
return k, v, ok
}
}
case []interface{}:
for _, v := range t {
if _, ok := SearchNested(v, key); ok {
return "", v, ok
}
}
}
return "", nil, false
}
// SearchNested searches a nested structure consisting of map[string]interface{}
// and []interface{} looking for a map with a specific key name.
// If found SearchNested returns the value associated with that key, true
// If the key is not found SearchNested returns nil, false
func SearchNested(obj interface{}, key string) (interface{}, bool) {
switch t := obj.(type) {
case map[string]interface{}:
if v, ok := t[key]; ok {
return v, ok
}
for _, v := range t {
if result, ok := SearchNested(v, key); ok {
return result, ok
}
}
case []interface{}:
for _, v := range t {
if result, ok := SearchNested(v, key); ok {
return result, ok
}
}
}
return nil, false
}
func handleSetHook(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] Api authentication failed in set new workflowhandler: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
// FIXME - check basic authentication
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Println(jsonPrettyPrint(string(body)))
var hook shuffle.Hook
err = json.Unmarshal(body, &hook)
if err != nil {
log.Printf("Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != hook.Owner && user.Role != "admin" && user.Role != "scheduler" {
log.Printf("Wrong user (%s) for hook %s", user.Username, hook.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if hook.Id != workflowId {
errorstring := fmt.Sprintf(`Id %s != %s`, hook.Id, workflowId)
log.Printf("Ids not matching: %s", errorstring)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, errorstring)))
return
}
// Verifies the hook JSON. Bad verification :^)
finished, errorstring := verifyHook(hook)
if !finished {
log.Printf("Error with hook: %s", errorstring)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, errorstring)))
return
}
// Get the ID to see whether it exists
// FIXME - use return and set READONLY fields (don't allow change from User)
ctx := context.Background()
_, err = shuffle.GetHook(ctx, workflowId)
if err != nil {
log.Printf("[WARNING] Failed getting hook %s (set): %s", workflowId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "Invalid ID"}`))
return
}
// Update the fields
err = shuffle.SetHook(ctx, hook)
if err != nil {
log.Printf("Failed setting hook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
// FIXME - some fields (e.g. status) shouldn't be writeable.. Meh
func verifyHook(hook shuffle.Hook) (bool, string) {
// required fields: Id, info.name, type, status, running
if hook.Id == "" {
return false, "Missing required field id"
}
if hook.Info.Name == "" {
return false, "Missing required field info.name"
}
// Validate type stuff
validTypes := []string{"webhook"}
found := false
for _, key := range validTypes {
if hook.Type == key {
found = true
break
}
}
if !found {
return false, fmt.Sprintf("Field type is invalid. Allowed: %s", strings.Join(validTypes, ", "))
}
// WEbhook specific
if hook.Type == "webhook" {
if hook.Info.Url == "" {
return false, "Missing required field info.url"
}
}
if hook.Status == "" {
return false, "Missing required field status"
}
validStatusFields := []string{"running", "stopped", "uninitialized"}
found = false
for _, key := range validStatusFields {
if hook.Status == key {
found = true
break
}
}
if !found {
return false, fmt.Sprintf("Field status is invalid. Allowed: %s", strings.Join(validStatusFields, ", "))
}
// Verify actions
if len(hook.Actions) > 0 {
existingIds := []string{}
for index, action := range hook.Actions {
if action.Type == "" {
return false, fmt.Sprintf("Missing required field actions.type at index %d", index)
}
if action.Name == "" {
return false, fmt.Sprintf("Missing required field actions.name at index %d", index)
}
if action.Id == "" {
return false, fmt.Sprintf("Missing required field actions.id at index %d", index)
}
// Check for duplicate IDs
for _, actionId := range existingIds {
if action.Id == actionId {
return false, fmt.Sprintf("actions.id %s at index %d already exists", actionId, index)
}
}
existingIds = append(existingIds, action.Id)
}
}
return true, "All items set"
//log.Printf("%#v", hook)
//Id string `json:"id" datastore:"id"`
//Info Info `json:"info" datastore:"info"`
//Transforms struct{} `json:"transforms" datastore:"transforms"`
//Actions []HookAction `json:"actions" datastore:"actions"`
//Type string `json:"type" datastore:"type"`
//Status string `json:"status" datastore:"status"`
//Running bool `json:"running" datastore:"running"`
}
func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
// FIXME - check basic authentication
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
jsonPrettyPrint(string(body))
var schedule shuffle.ScheduleOld
err = json.Unmarshal(body, &schedule)
if err != nil {
log.Printf("Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - check access etc
ctx := context.Background()
err = shuffle.SetSchedule(ctx, schedule)
if err != nil {
log.Printf("Failed setting schedule: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - get some real data?
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
return
}
func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
ctx := context.Background()
// FIXME: Schedule = trigger?
schedule, err := shuffle.GetSchedule(ctx, workflowId)
if err != nil {
log.Printf("Failed setting schedule: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//log.Printf("%#v", schedule.Translator[0])
b, err := json.Marshal(schedule)
if err != nil {
log.Printf("Failed marshalling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - get some real data?
resp.WriteHeader(200)
resp.Write([]byte(b))
return
}
// Starts a new webhook
func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in set new workflowhandler: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: IAM - Get workflow and check owner
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Admin required"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
ctx := context.Background()
err = shuffle.DeleteKey(ctx, "schedules", workflowId)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "Can't delete"}`))
return
}
// FIXME - remove schedule too
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "message": "Deleted webhook"}`))
}
// Starts a new webhook
func handleNewSchedule(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
randomValue := uuid.NewV4()
h := md5.New()
io.WriteString(h, randomValue.String())
newId := strings.ToLower(fmt.Sprintf("%X", h.Sum(nil)))
// FIXME - timestamp!
// FIXME - applocation - cloud function?
timeNow := int64(time.Now().Unix())
schedule := shuffle.ScheduleOld{
Id: newId,
AppInfo: shuffle.AppInfo{},
BaseAppLocation: "/home/frikky/git/shaffuru/tmp/apps",
CreationTime: timeNow,
LastModificationtime: timeNow,
LastRuntime: timeNow,
}
ctx := context.Background()
err := shuffle.SetSchedule(ctx, schedule)
if err != nil {
log.Printf("Failed setting hook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Println("Generating new schedule")
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "message": "Created new service"}`))
}
// Does the webhook
func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
// 1. Get callback data
// 2. Load the configuration
// 3. Execute the workflow
//cors := shuffle.HandleCors(resp, request)
//if cors {
// return
//}
log.Printf("[DEBUG] HOOKS: webhook callback: %s", request.URL.String())
if request.Method != "POST" {
request.Method = "POST"
}
if request.Body == nil {
stringReader := strings.NewReader("")
request.Body = ioutil.NopCloser(stringReader)
}
path := strings.Split(request.URL.String(), "/")
if len(path) < 4 {
log.Printf("[DEBUG] HOOKS: Invalid webhook path: %s", request.URL.String())
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
// 1. Get config with hookId
//fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId)
ctx := context.Background()
location := strings.Split(request.URL.String(), "/")
var hookId string
var queries string
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Couldn't handle location. Too short in webhook: %d", len(location))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
hookId = location[4]
}
if strings.Contains(hookId, "?") {
splitter := strings.Split(hookId, "?")
hookId = splitter[0]
if len(splitter) > 1 {
queries = splitter[1]
}
}
log.Printf("[DEBUG] HOOKS: Pre user agent check")
// Find user agent header
userAgent := request.Header.Get("User-Agent")
if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") {
log.Printf("[AUDIT] Blocking googlebot and microsoftbot for webhooks. UA: '%s'", userAgent)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`))
return
}
// ID: webhook_<UID>
if len(hookId) != 44 {
log.Printf("[INFO] Couldn't handle hookId. Too short in webhook: %d", len(hookId))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Hook ID not valid"}`))
return
}
hookId = hookId[8:len(hookId)]
//log.Printf("HookID: %s", hookId)
hook, err := shuffle.GetHook(ctx, hookId)
if err != nil {
log.Printf("[WARNING] HOOKS: Failed getting hook %s (callback): %s", hookId, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
//log.Printf("HOOK FOUND: %#v", hook)
// Execute the workflow
//executeWorkflow(resp, request)
//resp.WriteHeader(200)
//resp.Write([]byte(`{"success": true}`))
if hook.Status == "stopped" {
log.Printf("[WARNING] HOOKS: Not running %s because hook status is stopped", hook.Id)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Is it running?"}`)))
return
}
if len(hook.Workflows) == 0 {
log.Printf("[DEBUG] HOOKS: Not running because hook isn't connected to any workflows")
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflows are defined"}`)))
return
}
if hook.Environment == "cloud" {
log.Printf("[DEBUG] HOOKS: This should trigger in the cloud. Duplicate action allowed onprem.")
}
// Check auth
if len(hook.Auth) > 0 {
err = shuffle.CheckHookAuth(request, hook.Auth)
if err != nil {
log.Printf("[WARNING] Failed auth for hook %s: %s", hook.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Bad authentication headers"}`))
return
}
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[DEBUG] HOOKS: data read error: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(queries) > 0 && len(body) == 0 {
body = []byte(queries)
}
//log.Printf("BODY: %s", parsedBody)
// This is a specific fix for MSteams and may fix other things as well
// Scared whether it may stop other things though, but that's a future problem
// (famous last words)
//log.Printf("\n\nPARSEDBODY: %s", parsedBody)
parsedBody := shuffle.GetExecutionbody(body)
newBody := shuffle.ExecutionStruct{
Start: hook.Start,
ExecutionSource: "webhook",
ExecutionArgument: parsedBody,
}
if len(hook.Workflows) == 1 {
workflow, err := shuffle.GetWorkflow(ctx, hook.Workflows[0], true)
if err == nil {
for _, branch := range workflow.Branches {
if branch.SourceID == hook.Id {
log.Printf("[DEBUG] Found ID %s for hook", hook.Id)
if branch.DestinationID != hook.Start {
newBody.Start = branch.DestinationID
break
}
}
}
}
}
b, err := json.Marshal(newBody)
if err != nil {
log.Printf("[ERROR] HOOKS: Failed newBody marshaling for webhook: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
// Should wrap the response input Body as well?
for _, item := range hook.Workflows {
log.Printf("[INFO] Running webhook for workflow %s with startnode %s", item, hook.Start)
// This ID is empty to force it to get the webhook within the execution
workflow := shuffle.Workflow{
ID: "",
}
if len(hook.Start) == 0 {
log.Printf("[ERROR] HOOKS: No start node for hook %s - running with workflow default.", hook.Id)
//bodyWrapper = string(parsedBody)
}
newRequest := &http.Request{
URL: &url.URL{},
Method: "POST",
Body: ioutil.NopCloser(bytes.NewReader(b)),
}
// OrgId: activeOrgs[0].Id,
workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest, hook.OrgId)
if err == nil {
if hook.Version == "v2" {
timeout := 15
//if hook.VersionTimeout != 0 {
// timeout = hook.VersionTimeout
//}
log.Printf("[DEBUG] Waiting for Webhook response from %s for max %d seconds! Checking every 1 second. Hook ID: %s", workflowExecution.ExecutionId, timeout, hook.Id)
// Try every second for 15 seconds
for i := 0; i < timeout; i++ {
time.Sleep(1 * time.Second)
newExec, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
if err != nil {
log.Printf("[ERROR] Failed to get workflow execution: %s", err)
break
}
if newExec.Status != "EXECUTING" {
log.Printf("[INFO] Got response from webhook v2 of length '%d' <- %s", len(newExec.Result), newExec.ExecutionId)
resp.WriteHeader(200)
resp.Write([]byte(newExec.Result))
return
}
}
}
// Fallback
resp.WriteHeader(200)
if len(hook.CustomResponse) > 0 {
resp.Write([]byte(hook.CustomResponse))
} else {
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId)))
}
return
}
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp)))
}
log.Printf("[ERROR] HOOKS: END OF FUNCTION FOR '%s'. IF this is reached, something went wrong.", hook.Id)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to run workflow. Check logs."}`))
}
func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) {
if request.Method != "POST" {
request.Method = "POST"
}
if request.Body == nil {
stringReader := strings.NewReader("")
request.Body = ioutil.NopCloser(stringReader)
}
path := strings.Split(request.URL.String(), "/")
if len(path) < 4 {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := context.Background()
location := strings.Split(request.URL.String(), "/")
var pipelineId string
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[INFO] Couldn't handle location. Too short in pipeline: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
pipelineId = location[4]
}
userAgent := request.Header.Get("User-Agent")
if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") {
log.Printf("[AUDIT] Blocking googlebot and microsoftbot for pipelines. UA: '%s'", userAgent)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Google/Microsoft preview bots not allowed. Please change the useragent."}`))
return
}
if len(pipelineId) != 45 {
log.Printf("[INFO] Couldn't handle pipeline. Too short in pipeline: %d", len(pipelineId))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "pipeline ID not valid"}`))
return
}
pipelineId = pipelineId[9:]
pipeline, err := shuffle.GetPipeline(ctx, pipelineId)
if err != nil {
log.Printf("[WARNING] Failed getting pipeline %s (callback): %s", pipelineId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if pipeline.Status != "running" {
log.Printf("[WARNING] Not running %s because pipeline status is %s", pipeline.TriggerId, pipeline.Status)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The pipeline isn't running"}`)))
return
}
if pipeline.WorkflowId == "" {
log.Printf("[DEBUG] Not running because pipeline isn't connected to any workflows")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflows are defined"}`)))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[DEBUG] Body data error: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Parse concatenated JSON logs
jsonList, err := parseConcatenatedJSONLogs(string(body))
if err != nil {
log.Printf("[DEBUG] JSON parsing error: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
parsedBody, err := json.Marshal(jsonList)
if err != nil {
log.Printf("[ERROR] Failed to marshal jsonList: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
newBody := shuffle.ExecutionStruct{
Start: pipeline.StartNode,
ExecutionSource: "pipeline",
ExecutionArgument: string(parsedBody),
}
workflow, err := shuffle.GetWorkflow(ctx, pipeline.WorkflowId)
if err == nil {
for _, branch := range workflow.Branches {
if branch.SourceID == pipeline.TriggerId {
log.Printf("[DEBUG] Found ID %s for pipeline", pipeline.TriggerId)
if branch.DestinationID != pipeline.StartNode {
newBody.Start = branch.DestinationID
break
}
}
}
}
b, err := json.Marshal(newBody)
if err != nil {
log.Printf("[ERROR] Failed newBody marshaling for pipeline: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] Running pipeline for workflow %s with startnode %s", pipeline.WorkflowId, pipeline.StartNode)
newWorkflow := shuffle.Workflow{
ID: "",
}
if len(pipeline.StartNode) == 0 {
log.Printf("[WARNING] No start node for pipeline %s - running with workflow default.")
}
newRequest := &http.Request{
URL: &url.URL{},
Method: "POST",
Body: ioutil.NopCloser(bytes.NewReader(b)),
}
workflowExecution, executionResp, err := handleExecution(pipeline.WorkflowId, newWorkflow, newRequest, pipeline.OrgId)
if err == nil {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId)))
// Track Sigma rules
trackSigmaRules(ctx, pipeline.OrgId, jsonList)
return
}
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp)))
}
func parseConcatenatedJSONLogs(logs string) ([]map[string]interface{}, error) {
var jsonList []map[string]interface{}
decoder := json.NewDecoder(strings.NewReader(logs))
for decoder.More() {
var jsonObject map[string]interface{}
if err := decoder.Decode(&jsonObject); err != nil {
log.Printf("[WARNING] JSON decoding error: %s. Skipping this object.", err)
continue
}
jsonList = append(jsonList, jsonObject)
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return nil, fmt.Errorf("error after decoding all JSON objects: %v", err)
}
return jsonList, nil
}
func trackSigmaRules(ctx context.Context, orgId string, jsonList []map[string]interface{}) {
ruleCount := make(map[string]int)
for _, logEntry := range jsonList {
if rule, ok := logEntry["rule"].(map[string]interface{}); ok {
if ruleName, ok := rule["title"].(string); ok {
ruleCount[ruleName]++
}
}
}
for ruleName, count := range ruleCount {
shuffle.IncrementCache(ctx, orgId, ruleName, count)
log.Printf("[INFO] Rule %s incremented by %d", ruleName, count)
}
}
func executeCloudAction(action shuffle.CloudSyncJob, apikey string) error {
data, err := json.Marshal(action)
if err != nil {
log.Printf("Failed cloud webhook action marshalling: %s", err)
return err
}
syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync/handle_action", syncUrl)
client := shuffle.GetExternalClient(syncUrl)
req, err := http.NewRequest(
"POST",
syncUrl,
bytes.NewBuffer(data),
)
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
newresp, err := client.Do(req)
if err != nil {
return err
}
defer newresp.Body.Close()
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
return err
}
type Result struct {
Success bool `json:"success"`
Reason string `json:"reason"`
}
//log.Printf("Data: %s", string(respBody))
responseData := Result{}
err = json.Unmarshal(respBody, &responseData)
if err != nil {
return err
}
if !responseData.Success {
return errors.New(fmt.Sprintf("Cloud error from Shuffler: %s", responseData.Reason))
}
log.Printf("[INFO] Cloud action executed successfully for '%s'", action.Action)
return nil
}
func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) {
if request.Method != "GET" {
setSpecificSchedule(resp, request)
return
}
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
ctx := context.Background()
schedule, err := shuffle.GetSchedule(ctx, workflowId)
if err != nil {
log.Printf("Failed getting schedule: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//log.Printf("%#v", schedule.Translator[0])
b, err := json.Marshal(schedule)
if err != nil {
log.Printf("Failed marshalling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(b))
}
func loadYaml(fileLocation string) (ApiYaml, error) {
apiYaml := ApiYaml{}
yamlFile, err := ioutil.ReadFile(fileLocation)
if err != nil {
log.Printf("yamlFile.Get err: %s", err)
return ApiYaml{}, err
}
err = yaml.Unmarshal([]byte(yamlFile), &apiYaml)
if err != nil {
return ApiYaml{}, err
}
return apiYaml, nil
}
// This should ALWAYS come from an OUTPUT
func executeSchedule(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
ctx := context.Background()
log.Printf("[INFO] EXECUTING %s!", workflowId)
idConfig, err := shuffle.GetSchedule(ctx, workflowId)
if err != nil {
log.Printf("Error getting schedule: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
// Basically the src app
inputStrings := map[string]string{}
for _, item := range idConfig.Translator {
if item.Dst.Required == "false" {
log.Println("Skipping not required")
continue
}
if item.Src.Name == "" {
errorMsg := fmt.Sprintf("Required field %s has no source", item.Dst.Name)
log.Println(errorMsg)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, errorMsg)))
return
}
inputStrings[item.Dst.Name] = item.Src.Name
}
configmap := map[string]string{}
for _, config := range idConfig.AppInfo.SourceApp.Config {
configmap[config.Key] = config.Value
}
// FIXME - this wont work for everything lmao
functionName := strings.ToLower(idConfig.AppInfo.SourceApp.Action)
functionName = strings.Replace(functionName, " ", "_", 10)
cmdArgs := []string{
fmt.Sprintf("%s/%s/app.py", baseAppPath, "thehive"),
fmt.Sprintf("--referenceid=%s", workflowId),
fmt.Sprintf("--function=%s", functionName),
}
for key, value := range configmap {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", key, value))
}
// FIXME - processname
baseProcess := "python3"
log.Printf("Executing: %s %s", baseProcess, strings.Join(cmdArgs, " "))
execSubprocess(baseProcess, cmdArgs)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func execSubprocess(cmdName string, cmdArgs []string) error {
cmd := exec.Command(cmdName, cmdArgs...)
cmdReader, err := cmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err)
return err
}
scanner := bufio.NewScanner(cmdReader)
go func() {
for scanner.Scan() {
log.Printf("Out: %s\n", scanner.Text())
}
}()
err = cmd.Start()
if err != nil {
fmt.Fprintln(os.Stderr, "Error starting Cmd", err)
return err
}
err = cmd.Wait()
if err != nil {
fmt.Fprintln(os.Stderr, "Error waiting for Cmd", err)
return err
}
return nil
}
// This should ALWAYS come from an OUTPUT
func uploadWorkflowResult(resp http.ResponseWriter, request *http.Request) {
// Post to a key with random data?
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
// FIXME - check if permission AND whether it exists
// FIXME - validate ID as well
ctx := context.Background()
schedule, err := shuffle.GetSchedule(ctx, workflowId)
if err != nil {
log.Printf("Failed setting schedule %s: %s", workflowId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Should use generic interfaces and parse fields OR
// build temporary struct based on api.yaml of the app
data, err := parseWorkflowParameters(resp, request)
if err != nil {
log.Printf("Invalid params: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
// Get the actual fields
foldername := schedule.AppInfo.SourceApp.Foldername
curOutputType := schedule.AppInfo.SourceApp.Name
curOutputAppOutput := schedule.AppInfo.SourceApp.Action
curInputType := schedule.AppInfo.DestinationApp.Name
translatormap := schedule.Translator
if len(curOutputType) <= 0 {
log.Printf("Id %s is invalid. Missing sourceapp name", workflowId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
if len(foldername) == 0 {
foldername = strings.ToLower(curOutputType)
}
if len(curOutputAppOutput) <= 0 {
log.Printf("Id %s is invalid. Missing source output ", workflowId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
if len(curInputType) <= 0 {
log.Printf("Id %s is invalid. Missing destination name", workflowId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
// Needs to be used for parsing properly
// Might be dumb to have the yaml as a file too
yamlpath := fmt.Sprintf("%s/%s/api.yaml", baseAppPath, foldername)
curyaml, err := loadYaml(yamlpath)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
//validFields := []string{}
requiredFields := []string{}
optionalFields := []string{}
for _, output := range curyaml.Output {
if output.Name != curOutputAppOutput {
continue
}
for _, outputparam := range output.OutputParameters {
if outputparam.Required == "true" {
if outputparam.Schema.Type == "string" {
requiredFields = append(requiredFields, outputparam.Name)
} else {
log.Printf("Outputparam schematype %s is not implemented.", outputparam.Schema.Type)
}
} else {
optionalFields = append(optionalFields, outputparam.Name)
}
}
// Wont reach here unless it's the right one
break
}
// Checks whether ALL required fields are filled
for _, fieldname := range requiredFields {
if data[fieldname] == nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Field %s is required"}`, fieldname)))
return
} else {
log.Printf("%s: %s", fieldname, data[fieldname])
}
}
// FIXME
// Verify whether it can be sent from the source to destination here
// Save to DB or send it straight? Idk
// Use e.g. google pubsub if cloud and maybe kafka locally
// FIXME - add more types :)
sourcedatamap := map[string]string{}
for key, value := range data {
switch v := value.(type) {
case string:
sourcedatamap[key] = value.(string)
default:
log.Printf("unexpected type %T", v)
}
}
log.Println(data)
log.Println(requiredFields)
log.Println(translatormap)
log.Println(sourcedatamap)
outputmap := map[string]string{}
for _, translator := range translatormap {
if translator.Src.Type == "static" {
log.Printf("%s = %s", translator.Dst.Name, translator.Src.Value)
outputmap[translator.Dst.Name] = translator.Src.Value
} else {
log.Printf("%s = %s", translator.Dst.Name, translator.Src.Name)
outputmap[translator.Dst.Name] = sourcedatamap[translator.Src.Name]
}
}
configmap := map[string]string{}
for _, config := range schedule.AppInfo.DestinationApp.Config {
configmap[config.Key] = config.Value
}
// FIXME - add function to run
// FIXME - add reference somehow
// FIXME - add apikey somehow
// Just package and run really?
// FIXME - generate from sourceapp
outputmap["function"] = "create_alert"
cmdArgs := []string{
fmt.Sprintf("%s/%s/app.py", baseAppPath, foldername),
}
for key, value := range outputmap {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", key, value))
}
// COnfig map!
for key, value := range configmap {
cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", key, value))
}
outputmap["referenceid"] = workflowId
baseProcess := "python3"
log.Printf("Executing: %s %s", baseProcess, strings.Join(cmdArgs, " "))
execSubprocess(baseProcess, cmdArgs)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
//dst: {name: "title", required: "true", type: "string"}
//
//"title": "symptomDescription",
//"description": "detailedDescription",
//"type": "ticketType",
//"sourceRef": "ticketId"
//"name": "secureworks",
//"id": "e07910a06a086c83ba41827aa00b26ed",
//"description": "I AM SECUREWORKS DESC",
//"action": "Get Tickets",
//"config": {}
//"name": "thehive",
// "id": "e07910a06a086c83ba41827aa00b26ef",
// "description": "I AM thehive DESC",
// "action": "Add ticket",
// "config": [{
// "key": "http://localhost:9000",
// "value": "kZJmmn05j8wndOGDGvKg/D9eKub1itwO"
// }]
func findValidScheduleAppFolders(rootAppFolder string) ([]string, error) {
rootFiles, err := ioutil.ReadDir(rootAppFolder)
if err != nil {
return []string{}, err
}
invalidRootFiles := []string{}
invalidRootFolders := []string{}
invalidAppFolders := []string{}
validAppFolders := []string{}
// This is dumb
allowedLanguages := []string{"py", "go"}
for _, rootfile := range rootFiles {
if !rootfile.IsDir() {
invalidRootFiles = append(invalidRootFiles, rootfile.Name())
continue
}
appFolderLocation := fmt.Sprintf("%s/%s", rootAppFolder, rootfile.Name())
appFiles, err := ioutil.ReadDir(appFolderLocation)
if err != nil {
// Invalid app folder (deleted within a few MS lol)
invalidRootFolders = append(invalidRootFolders, rootfile.Name())
continue
}
yamlFileDone := false
appFileExists := false
for _, appfile := range appFiles {
if appfile.Name() == "api.yaml" {
err := validateAppYaml(
fmt.Sprintf("%s/%s", appFolderLocation, appfile.Name()),
)
if err != nil {
log.Printf("Error in %s: %s", fmt.Sprintf("%s/%s", rootfile.Name(), appfile.Name()), err)
break
}
log.Printf("YAML FOR %s: %s IS VALID!!", rootfile.Name(), appfile.Name())
yamlFileDone = true
}
for _, language := range allowedLanguages {
if appfile.Name() == fmt.Sprintf("app.%s", language) {
log.Printf("Appfile found for %s", rootfile.Name())
appFileExists = true
break
}
}
}
if !yamlFileDone || !appFileExists {
invalidAppFolders = append(invalidAppFolders, rootfile.Name())
} else {
validAppFolders = append(validAppFolders, rootfile.Name())
}
}
log.Printf("Invalid rootfiles: %s", strings.Join(invalidRootFiles, ", "))
log.Printf("Invalid rootfolders: %s", strings.Join(invalidRootFolders, ", "))
log.Printf("Invalid appfolders: %s", strings.Join(invalidAppFolders, ", "))
log.Printf("\n=== VALID appfolders ===\n* %s", strings.Join(validAppFolders, "\n"))
return validAppFolders, err
}
func validateInputOutputYaml(appType string, apiYaml ApiYaml) error {
if appType == "input" {
for index, input := range apiYaml.Input {
if input.Name == "" {
return errors.New(fmt.Sprintf("YAML field name doesn't exist in index %d of Input", index))
}
if input.Description == "" {
return errors.New(fmt.Sprintf("YAML field description doesn't exist in index %d of Input", index))
}
for paramindex, param := range input.InputParameters {
if param.Name == "" {
return errors.New(fmt.Sprintf("YAML field name doesn't exist in Input %s with index %d", input.Name, paramindex))
}
if param.Description == "" {
return errors.New(fmt.Sprintf("YAML field description doesn't exist in Input %s with index %d", input.Name, index))
}
if param.Schema.Type == "" {
return errors.New(fmt.Sprintf("YAML field schema.type doesn't exist in Input %s with index %d", input.Name, index))
}
}
}
}
return nil
}
func validateAppYaml(fileLocation string) error {
/*
Requires:
name, description, app_version, contact_info (name), types
*/
apiYaml, err := loadYaml(fileLocation)
if err != nil {
return err
}
// Validate fields
if apiYaml.Name == "" {
return errors.New("YAML field name doesn't exist")
}
if apiYaml.Description == "" {
return errors.New("YAML field description doesn't exist")
}
if apiYaml.AppVersion == "" {
return errors.New("YAML field app_version doesn't exist")
}
if apiYaml.ContactInfo.Name == "" {
return errors.New("YAML field contact_info.name doesn't exist")
}
if len(apiYaml.Types) == 0 {
return errors.New("YAML field types doesn't exist")
}
// Validate types (input/ouput)
validTypes := []string{"input", "output"}
for _, appType := range apiYaml.Types {
// Validate in here lul
for _, validType := range validTypes {
if appType == validType {
err = validateInputOutputYaml(appType, apiYaml)
if err != nil {
return err
}
break
}
}
}
return nil
}
func setBadMemcache(ctx context.Context, path string) {
// Add to cache if it doesn't exist
//item := &memcache.Item{
// Key: path,
// Value: []byte(`{"success": false}`),
// Expiration: time.Minute * 60,
//}
//if err := memcache.Add(ctx, item); err == memcache.ErrNotStored {
// if err := memcache.Set(ctx, item); err != nil {
// log.Printf("Error setting item: %v", err)
// }
//} else if err != nil {
// log.Printf("error adding item: %v", err)
//} else {
// log.Printf("Set cache for %s", item.Key)
//}
}
type Result struct {
Success bool `json:"success"`
Reason string `json:"reason"`
List []string `json:"list"`
}
// r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS")
func getOpenapi(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
_, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in validate swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var id string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
id = location[4]
}
if len(id) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - FIX AUTH WITH APP
ctx := context.Background()
//_, err = shuffle.GetApp(ctx, id)
//if err == nil {
// log.Println("You're supposed to be able to continue now.")
//}
parsedApi, err := shuffle.GetOpenApiDatastore(ctx, id)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] API LENGTH GET FOR OPENAPI %s: %d, ID: %s", id, len(parsedApi.Body), id)
parsedApi.Success = true
data, err := json.Marshal(parsedApi)
if err != nil {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling parsed swagger: %s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write(data)
}
func handleSwaggerValidation(body []byte) (shuffle.ParsedOpenApi, error) {
type versionCheck struct {
Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"`
SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"`
OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"`
}
//body = []byte(`swagger: "2.0"`)
//body = []byte(`swagger: '1.0'`)
//newbody := string(body)
//newbody = strings.TrimSpace(newbody)
//body = []byte(newbody)
//log.Println(string(body))
//tmpbody, err := yaml.YAMLToJSON(body)
//log.Println(err)
//log.Println(string(tmpbody))
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version versionCheck
parsed := shuffle.ParsedOpenApi{}
swaggerdata := []byte{}
idstring := ""
isJson := false
err := json.Unmarshal(body, &version)
if err != nil {
//log.Printf("Json err: %s", err)
err = yaml.Unmarshal(body, &version)
if err != nil {
log.Printf("[WARNING] Yaml error (1): %s", err)
} else {
//log.Printf("Successfully parsed YAML!")
}
} else {
isJson = true
//log.Printf("[DEBUG] Successfully parsed JSON!")
}
if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 {
version.Swagger = version.SwaggerVersion
}
if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") {
//log.Println("Handling v3 API")
swaggerLoader := openapi3.NewSwaggerLoader()
swaggerLoader.IsExternalRefsAllowed = true
swaggerv3, err := swaggerLoader.LoadSwaggerFromData(body)
if err != nil {
log.Printf("Failed parsing OpenAPI: %s", err)
return shuffle.ParsedOpenApi{}, err
}
swaggerdata, err = json.Marshal(swaggerv3)
if err != nil {
log.Printf("Failed unmarshaling v3 data: %s", err)
return shuffle.ParsedOpenApi{}, err
}
hasher := md5.New()
hasher.Write(swaggerdata)
idstring = hex.EncodeToString(hasher.Sum(nil))
} else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") {
// Convert
//log.Println("Handling v2 API")
var swagger openapi2.Swagger
//log.Println(string(body))
err = json.Unmarshal(body, &swagger)
if err != nil {
//log.Printf("Json error? %s", err)
err = yaml.Unmarshal(body, &swagger)
if err != nil {
log.Printf("[WARNING] Yaml error (2): %s", err)
return shuffle.ParsedOpenApi{}, err
} else {
//log.Printf("Valid yaml!")
}
}
swaggerv3, err := openapi2conv.ToV3Swagger(&swagger)
if err != nil {
log.Printf("Failed converting from openapi2 to 3: %s", err)
return shuffle.ParsedOpenApi{}, err
}
swaggerdata, err = json.Marshal(swaggerv3)
if err != nil {
log.Printf("Failed unmarshaling v3 data: %s", err)
return shuffle.ParsedOpenApi{}, err
}
hasher := md5.New()
hasher.Write(swaggerdata)
idstring = hex.EncodeToString(hasher.Sum(nil))
}
if len(swaggerdata) > 0 {
body = swaggerdata
}
// Overwrite with new json data
_ = isJson
body = swaggerdata
// Parsing it to swagger 3
parsed = shuffle.ParsedOpenApi{
ID: idstring,
Body: string(body),
Success: true,
}
return parsed, err
}
func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, skipEdit bool) {
type Test struct {
Editing bool `json:"editing" datastore:"editing"`
Id string `json:"id" datastore:"id"`
Image string `json:"image" datastore:"image"`
Body string `json:"body" datastore:"body"`
}
var test Test
err := json.Unmarshal(body, &test)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling in swagger build: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Get an identifier
hasher := md5.New()
hasher.Write(body)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
if test.Editing && len(user.Id) > 0 && skipEdit != true {
// Quick verification test
ctx := context.Background()
app, err := shuffle.GetApp(ctx, test.Id, user, false)
if err != nil {
log.Printf("[ERROR] Error getting app when editing: %s", app.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id == app.Owner || (user.Role == "admin" && user.ActiveOrg.Id == app.ReferenceOrg) || shuffle.ArrayContains(app.Contributors, user.Id) {
log.Printf("[DEBUG] Editing app %s with user %s (%s) in org %s", test.Id, user.Username, user.Id, user.ActiveOrg.Id)
} else {
log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "You don't have permissions to edit this app. Contact support@shuffler.io if this persists."}`))
return
}
log.Printf("[INFO] %s is EDITING APP WITH ID %s and md5 %s", user.Id, app.ID, newmd5)
newmd5 = app.ID
}
// Generate new app integration (bump version)
// Test = client side with fetch?
ctx := context.Background()
swaggerLoader := openapi3.NewSwaggerLoader()
swaggerLoader.IsExternalRefsAllowed = true
swagger, err := swaggerLoader.LoadSwaggerFromData(body)
if err != nil {
log.Printf("[ERROR] Swagger validation error: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`))
return
}
if swagger.Info == nil {
log.Printf("[ERORR] Info is nil in swagger?")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Info not parsed"}`))
return
}
swagger.Info.Title = shuffle.FixFunctionName(swagger.Info.Title, swagger.Info.Title, false)
if strings.Contains(swagger.Info.Title, " ") {
swagger.Info.Title = strings.Replace(swagger.Info.Title, " ", "_", -1)
}
basePath, err := shuffle.BuildStructure(swagger, newmd5)
if err != nil {
log.Printf("[WARNING] Failed to build base structure: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed building baseline structure"}`))
return
}
//log.Printf("Should generate yaml")
swagger, api, pythonfunctions, err := shuffle.GenerateYaml(swagger, newmd5)
if err != nil {
log.Printf("[WARNING] Failed building and generating yaml (buildapp): %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed building and parsing yaml"}`))
return
}
// FIXME: CHECK IF SAME NAME AS NORMAL APP
// Can't overwrite existing normal app
workflowApps, err := shuffle.GetPrioritizedApps(ctx, user)
if err != nil {
log.Printf("[WARNING] Failed getting all workflow apps from database to verify: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to verify existence"}`))
return
}
// Same name only?
lowerName := strings.ToLower(swagger.Info.Title)
for _, app := range workflowApps {
if app.Downloaded && !app.Generated && strings.ToLower(app.Name) == lowerName {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Normal app with name %s already exists. Delete it first."}`, swagger.Info.Title)))
return
}
}
if api.Owner == "" {
api.Owner = user.Id
}
if len(api.ReferenceOrg) == 0 {
api.ReferenceOrg = user.ActiveOrg.Id
}
if len(test.Image) > 0 {
api.SmallImage = test.Image
api.LargeImage = test.Image
}
err = shuffle.DumpApi(basePath, api)
if err != nil {
log.Printf("[WARNING] Failed dumping yaml: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed dumping yaml"}`))
return
}
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, newmd5)
classname := strings.Replace(identifier, " ", "", -1)
classname = strings.Replace(classname, "-", "", -1)
parsedCode, err := shuffle.DumpPython(basePath, classname, swagger.Info.Version, pythonfunctions)
if err != nil {
log.Printf("Failed dumping python: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed dumping appcode"}`))
return
}
identifier = strings.Replace(identifier, " ", "-", -1)
identifier = strings.Replace(identifier, "_", "-", -1)
log.Printf("[INFO] Successfully parsed %s. Proceeding to docker container", identifier)
// Now that the baseline is setup, we need to make it into a cloud function
// 1. Upload the API to datastore for use
// 2. Get code from baseline/app_base.py & baseline/static_baseline.py
// 3. Stitch code together from these two + our new app
// 4. Zip the folder to cloud storage
// 5. Upload as cloud function
// 1. Upload the API to datastore
err = shuffle.DeployAppToDatastore(ctx, api)
//func DeployAppToDatastore(ctx context.Context, workflowapp WorkflowApp, bucketName string) error {
if err != nil {
log.Printf("[ERROR] Failed adding app to db: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed adding app to db: %s"}`, err)))
return
}
// 2. Get all the required code
appbase, staticBaseline, err := shuffle.GetAppbase()
if err != nil {
log.Printf("[ERROR] Failed getting appbase: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting appbase code"}`))
return
}
// Have to do some quick checks of the python code (:
_, parsedCode = shuffle.FormatAppfile(parsedCode)
fixedAppbase := shuffle.FixAppbase(appbase)
runner := shuffle.GetRunnerOnprem(classname)
// 2. Put it together
stitched := string(staticBaseline) + strings.Join(fixedAppbase, "\n") + parsedCode + string(runner)
//log.Println(stitched)
// 3. Zip and stream it directly in the directory
_, err = shuffle.StreamZipdata(ctx, identifier, stitched, shuffle.GetAppRequirements(), "")
if err != nil {
log.Printf("[ERROR] Zipfile error: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to build zipfile"}`))
return
}
log.Printf("[INFO] Successfully stitched ZIPFILE for %s", identifier)
// Copy baseline Dockerfile to build directory
dockerfileSource := "../app_gen/python-lib/baseline/Dockerfile"
dockerfileDestination := fmt.Sprintf("%s/Dockerfile", basePath)
// Read and copy the baseline Dockerfile
dockerfileContent, err := ioutil.ReadFile(dockerfileSource)
if err != nil {
foundDockerfile := shuffle.GetBaseDockerfile()
if len(foundDockerfile) > 0 {
dockerfileContent = foundDockerfile
} else {
log.Printf("[ERROR] Failed to read baseline Dockerfile: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to read baseline Dockerfile"}`))
return
}
}
err = ioutil.WriteFile(dockerfileDestination, dockerfileContent, 0644)
if err != nil {
log.Printf("[ERROR] Failed to copy Dockerfile to build directory: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to copy Dockerfile"}`))
return
}
// Verify the Dockerfile was created
if _, err := os.Stat(dockerfileDestination); os.IsNotExist(err) {
log.Printf("[ERROR] Dockerfile does not exist at destination: %s", dockerfileDestination)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Dockerfile was not created properly"}`))
return
}
// 4. Build the image locally.
// FIXME: Should be moved to a local docker registry
dockerLocation := fmt.Sprintf("%s/Dockerfile", basePath)
log.Printf("[INFO] Dockerfile: %s", dockerLocation)
versionName := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(api.Name, " ", "-")), api.AppVersion)
dockerTags := []string{
fmt.Sprintf("%s:%s", baseDockerName, identifier),
fmt.Sprintf("%s:%s", baseDockerName, versionName),
}
found := false
foundNumber := 0
log.Printf("[INFO] Checking for api with ID %s", newmd5)
for appCounter, app := range user.PrivateApps {
if app.ID == api.ID {
found = true
foundNumber = appCounter
break
} else if app.Name == api.Name && app.AppVersion == api.AppVersion {
found = true
foundNumber = appCounter
break
} else if app.PrivateID == test.Id && test.Editing {
found = true
foundNumber = appCounter
break
}
}
// Updating the user with the new app so that it can easily be retrieved
if !found {
user.PrivateApps = append(user.PrivateApps, api)
} else {
user.PrivateApps[foundNumber] = api
}
if len(user.Id) > 0 {
err = shuffle.SetUser(ctx, &user, true)
if err != nil {
log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err)
//resp.WriteHeader(500)
//resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Failed updating user"}`)))
//return
}
}
//log.Printf("DO I REACH HERE WHEN SAVING?")
parsed := shuffle.ParsedOpenApi{
ID: newmd5,
Body: string(body),
}
if !shuffle.ArrayContains(api.Contributors, user.Id) {
api.Contributors = append(api.Contributors, user.Id)
}
shuffle.SetAppRevision(ctx, api)
log.Printf("[INFO] API LENGTH FOR %s: %d, ID: %s", api.Name, len(parsed.Body), newmd5)
// FIXME: Might cause versioning issues if we re-use the same!!
// FIXME: Need a way to track different versions of the same app properly.
// Hint: Save API.id somewhere, and use newmd5 to save latest version
if len(user.Id) > 0 {
err = shuffle.SetOpenApiDatastore(ctx, newmd5, parsed)
if err != nil {
log.Printf("[ERROR] Failed saving app %s to database: %s", newmd5, err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%"}`, err)))
return
}
shuffle.SetOpenApiDatastore(ctx, api.ID, parsed)
} else {
//log.Printf("
}
// Backup every single one
/*
err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
*/
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
// Doing this last to ensure we can copy the docker image over
// even though builds fail
err = buildImage(dockerTags, dockerLocation)
if err != nil {
log.Printf("[ERROR] Docker build error: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error in Docker build: %s"}`, err)))
return
}
if len(user.ActiveOrg.Id) > 0 {
org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org during image build (%s): %s", user.ActiveOrg.Id, err)
} else {
log.Printf("[INFO] Successfully uploaded app %s to org %s (2). Validating and distributing image to available environments in org.", api.ID, org.Id)
imagenames := []string{
fmt.Sprintf("%s_%s", api.Name, api.AppVersion),
fmt.Sprintf("%s_%s", api.Name, test.Id),
}
err = shuffle.DistributeAppToEnvironments(ctx, *org, imagenames)
if err != nil {
log.Printf("[ERROR] Failed distributing app to environments: %s", err)
}
}
}
log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID)
if len(user.Id) > 0 {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, api.ID)))
}
org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org during image build (%s): %s", user.ActiveOrg.Id, err)
} else {
imagenames := []string{
fmt.Sprintf("%s_%s", api.Name, api.AppVersion),
fmt.Sprintf("%s_%s", api.Name, api.ID),
}
err = shuffle.DistributeAppToEnvironments(ctx, *org, imagenames)
if err != nil {
log.Printf("[ERROR] Failed distributing app to environments: %s", err)
}
}
}
// Creates an app from the app builder
func verifySwagger(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
//log.Printf("[INFO] TRY TO SET APP TO LIVE!!!")
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in verify swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to check swagger doc: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
buildSwaggerApp(resp, body, user, false)
}
// Hotloads new apps from a folder
func handleAppHotload(ctx context.Context, location string, forceUpdate bool) error {
basepath := "base"
fs, err := shuffle.CreateFs(basepath, location)
if err != nil {
log.Printf("Failed memfs creation - probably bad path: %s", err)
return errors.New(fmt.Sprintf("Failed to find directory %s", location))
} else {
log.Printf("[INFO] Memfs creation from %s done", location)
}
dir, err := fs.ReadDir("")
if err != nil {
log.Printf("[WARNING] Failed reading folder: %s", err)
return err
}
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, false)
if err != nil {
log.Printf("[WARNING] Githubfolders error: %s", err)
return err
}
cacheKey := fmt.Sprintf("workflowapps-sorted")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-100")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
//shuffle.DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
return nil
}
func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executionArgument string) error {
ctx := context.Background()
// 1. Get the workflow
// 2. Execute it with the data
workflow, err := shuffle.GetWorkflow(ctx, workflowId)
if err != nil {
return err
}
// FIXME: Handle auth
_ = workflow
parsedArgument := executionArgument
newExec := shuffle.ExecutionRequest{
ExecutionSource: executionSource,
ExecutionArgument: parsedArgument,
}
var execution shuffle.ExecutionRequest
err = json.Unmarshal([]byte(parsedArgument), &execution)
if err == nil {
//log.Printf("[INFO] FOUND EXEC %#v", execution)
if len(execution.ExecutionArgument) > 0 {
parsedArgument := strings.Replace(string(execution.ExecutionArgument), "\\\"", "\"", -1)
log.Printf("New exec argument: %s", execution.ExecutionArgument)
if strings.HasPrefix(parsedArgument, "{") && strings.HasSuffix(parsedArgument, "}") {
log.Printf("\nData is most likely JSON from %s\n", newExec.ExecutionSource)
}
newExec.ExecutionArgument = parsedArgument
}
} else {
log.Printf("Unmarshal issue: %s", err)
}
if len(startNode) > 0 {
newExec.Start = startNode
}
b, err := json.Marshal(newExec)
if err != nil {
log.Printf("Failed marshal")
return err
}
//log.Println(string(b))
newRequest := &http.Request{
URL: &url.URL{},
Method: "POST",
Body: ioutil.NopCloser(bytes.NewReader(b)),
}
_, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest, workflow.OrgId)
return err
}
func handleCloudJob(job shuffle.CloudSyncJob) error {
ctx := context.Background()
// May need authentication in all of these..?
log.Printf("[INFO] Handle job with type %s and action %s", job.Type, job.Action)
shuffle.IncrementCache(ctx, job.OrgId, "org_sync_actions")
if job.Type == "outlook" {
if job.Action == "execute" {
// FIXME: Get the email
ctx := context.Background()
maildata := shuffle.MailDataOutlook{}
err := json.Unmarshal([]byte(job.ThirdItem), &maildata)
if err != nil {
log.Printf("Maildata unmarshal error: %s", err)
return err
}
hookId := job.Id
hook, err := shuffle.GetTriggerAuth(ctx, hookId)
if err != nil {
log.Printf("[INFO] Failed getting trigger %s (callback cloud): %s", hookId, err)
return err
}
backendPort := os.Getenv("BACKEND_PORT")
if backendPort == "" {
backendPort = "5001"
}
redirectDomain := fmt.Sprintf("localhost:%s", backendPort)
redirectUrl := fmt.Sprintf("http://%s/api/v1/triggers/outlook/register", redirectDomain)
outlookClient, _, err := shuffle.GetOutlookClient(ctx, "", hook.OauthToken, redirectUrl)
if err != nil {
log.Printf("Oauth client failure - triggerauth: %s", err)
return err
}
emails, err := shuffle.GetOutlookEmail(outlookClient, maildata)
//log.Printf("EMAILS: %d", len(emails))
//log.Printf("INSIDE GET OUTLOOK EMAIL!: %#v, %s", emails, err)
//type FullEmail struct {
email := shuffle.FullEmail{}
if len(emails) == 1 {
email = emails[0]
}
emailBytes, err := json.Marshal(email)
if err != nil {
log.Printf("[INFO] Failed email marshaling: %s", err)
return err
}
log.Printf("[INFO] Should handle outlook webhook for workflow %s with start node %s and data of length %d", job.PrimaryItemId, job.SecondaryItem, len(job.ThirdItem))
err = handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "outlook", string(emailBytes))
if err != nil {
log.Printf("[WARNING] Failed executing workflow from cloud outlook hook: %s", err)
} else {
log.Printf("[INFO] Successfully executed workflow from cloud outlook hook!")
}
}
} else if job.Type == "webhook" {
if job.Action == "execute" {
log.Printf("[INFO] Should handle normal webhook for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "webhook", job.ThirdItem)
if err != nil {
log.Printf("[INFO] Failed executing workflow from cloud hook: %s", err)
} else {
log.Printf("[INFO] Successfully executed workflow from cloud hook!")
}
}
} else if job.Type == "schedule" {
if job.Action == "execute" {
log.Printf("[INFO] Should handle schedule for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "schedule", job.ThirdItem)
if err != nil {
log.Printf("[INFO] Failed executing workflow from cloud schedule: %s", err)
} else {
log.Printf("[INFO] Successfully executed workflow from cloud schedule")
}
}
} else if job.Type == "email_trigger" {
if job.Action == "execute" {
log.Printf("[INFO] Should handle email for workflow %s with start node %s and data %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
err := handleCloudExecutionOnprem(job.PrimaryItemId, job.SecondaryItem, "email_trigger", job.ThirdItem)
if err != nil {
log.Printf("Failed executing workflow from email trigger: %s", err)
} else {
log.Printf("Successfully executed workflow from cloud email trigger")
}
}
} else if job.Type == "user_input" {
if job.Action == "continue" {
log.Printf("[INFO] Should handle user_input CONTINUE for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
// FIXME: Handle authorization
ctx := context.Background()
workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem)
if err != nil {
return err
}
if job.PrimaryItemId != workflowExecution.Workflow.ID {
return errors.New("Bad workflow ID when stopping execution.")
}
workflowExecution.Status = "EXECUTING"
err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true)
if err != nil {
return err
}
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=true", syncUrl, job.PrimaryItemId, job.FourthItem, job.SecondaryItem, job.ThirdItem)
newRequest, err := http.NewRequest(
"GET",
fullUrl,
nil,
)
if err != nil {
log.Printf("Failed continuing workflow in request builder: %s", err)
return err
}
_, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest, job.OrgId)
if err != nil {
log.Printf("Failed continuing workflow from cloud user_input: %s", err)
return err
} else {
log.Printf("Successfully executed workflow from cloud user_input")
}
} else if job.Action == "stop" {
log.Printf("Should handle user_input STOP for workflow %s with start node %s and execution ID %s", job.PrimaryItemId, job.SecondaryItem, job.ThirdItem)
ctx := context.Background()
workflowExecution, err := shuffle.GetWorkflowExecution(ctx, job.ThirdItem)
if err != nil {
return err
}
if job.PrimaryItemId != workflowExecution.Workflow.ID {
return errors.New("Bad workflow ID when stopping execution.")
}
/*
if job.FourthItem != workflowExecution.Authorization {
return errors.New("Bad authorization when stopping execution.")
}
*/
newResults := []shuffle.ActionResult{}
for _, result := range workflowExecution.Results {
if result.Action.AppName == "User Input" && result.Result == "Waiting for user feedback based on configuration" {
result.Status = "ABORTED"
result.Result = "Aborted manually by user."
}
newResults = append(newResults, result)
}
workflowExecution.Results = newResults
workflowExecution.Status = "ABORTED"
err = shuffle.SetWorkflowExecution(ctx, *workflowExecution, true)
if err != nil {
return err
}
log.Printf("Successfully updated user input to aborted.")
}
} else {
log.Printf("No handler for type %s and action %s", job.Type, job.Action)
}
return nil
}
// Handles jobs from remote (cloud)
func remoteOrgJobController(org shuffle.Org, body []byte) error {
type retStruct struct {
Success bool `json:"success"`
Reason string `json:"reason"`
Jobs []shuffle.CloudSyncJob `json:"jobs"`
SyncFeatures shuffle.SyncFeatures `json:"sync_features"`
Subscriptions []shuffle.PaymentSubscription `json:"subscriptions"`
Licensed bool `json:"licensed"`
CloudSyncUrl string `json:"cloud_sync_url,omitempty"`
}
responseData := retStruct{}
err := json.Unmarshal(body, &responseData)
if err != nil {
return err
}
ctx := context.Background()
if !responseData.Success {
log.Printf("[WARNING] Should stop org job controller because no success?")
if strings.Contains(strings.ToLower(responseData.Reason), "bad apikey") || strings.Contains(responseData.Reason, "Error getting the organization") || strings.Contains(responseData.Reason, "Organization isn't syncing") {
log.Printf("[WARNING] Remote error; Bad apikey or org error. Stopping sync for org: %s", responseData.Reason)
if value, exists := scheduledOrgs[org.Id]; exists {
// Looks like this does the trick? Hurr
log.Printf("[INFO] STOPPING ORG SCHEDULE for: %s", org.Id)
value.Lock()
} else {
log.Printf("[INFO] Failed finding the schedule for org %s (%s)", org.Name, org.Id)
}
org, err := shuffle.GetOrg(ctx, org.Id)
if err != nil {
log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err)
return err
}
// Just in case
org, err = handleStopCloudSync(syncUrl, *org)
if err != nil {
log.Printf("[ERROR] Failed stopping cloud sync remotely: %s", err)
}
org.SyncConfig.Interval = 0
org.CloudSync = false
org.SyncConfig.Apikey = ""
startDate := time.Now().Unix()
org.SyncFeatures.Webhook = shuffle.SyncData{Active: false, Type: "trigger", Name: "Webhook", StartDate: startDate}
org.SyncFeatures.UserInput = shuffle.SyncData{Active: false, Type: "trigger", Name: "User Input", StartDate: startDate}
org.SyncFeatures.EmailTrigger = shuffle.SyncData{Active: false, Type: "action", Name: "Email Trigger", StartDate: startDate}
org.SyncFeatures.Schedules = shuffle.SyncData{Active: false, Type: "trigger", Name: "Schedule", StartDate: startDate, Limit: 0}
org.SyncFeatures.SendMail = shuffle.SyncData{Active: false, Type: "action", Name: "Send Email", StartDate: startDate, Limit: 0}
org.SyncFeatures.SendSms = shuffle.SyncData{Active: false, Type: "action", Name: "Send SMS", StartDate: startDate, Limit: 0}
org.CloudSyncActive = false
err = shuffle.SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed setting organization when stopping sync: %s", err)
} else {
log.Printf("[INFO] Successfully STOPPED org cloud sync for %s (%s)", org.Name, org.Id)
}
return nil
}
return errors.New("[ERROR] Remote job handler issues.")
}
if len(responseData.Jobs) > 0 {
//log.Printf("[INFO] Remote JOB ret: %s", string(body))
log.Printf("Got job with reason %s and %d job(s)", responseData.Reason, len(responseData.Jobs))
}
cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id)
featuresBytes, err := json.Marshal(responseData.SyncFeatures)
if err != nil {
log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err)
} else {
shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800)
}
subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id)
subscriptionsBytes, err := json.Marshal(responseData.Subscriptions)
if err != nil {
log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err)
} else {
shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800)
}
licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id)
licensedBytes, err := json.Marshal(responseData.Licensed)
if err != nil {
log.Printf("[ERROR] Failed to marshal Licensed for cache: %s", err)
} else {
shuffle.SetCache(ctx, licenseCacheKey, licensedBytes, 1800)
}
for _, job := range responseData.Jobs {
err = handleCloudJob(job)
if err != nil {
log.Printf("[ERROR] Failed job from cloud: %s", err)
}
}
return nil
}
func remoteOrgJobHandler(org shuffle.Org, interval int) error {
// Check if it's 1 in 10 (10% chance random)
backupJob := shuffle.BackupJob{}
// Check if workflow backup is active
// Check if app backup is active
ctx := context.Background()
foundUser := org.Users[0]
for _, user := range org.Users {
if user.Role == "admin" {
foundUser = user
break
}
}
// Check if it's 1/20 times (600 seconds - 10 min on average)
// Only problem: May take time to sync the first time, which is annoying
// This is to ensure that we don't spam the shuffle cloud servers with a lot of data
shouldBackupData := false
randomNumber := rand.Intn(20)
if randomNumber == 0 {
shouldBackupData = true
}
// Just to prevent it from spamming large outbound requests
if shouldBackupData {
if org.SyncConfig.WorkflowBackup {
workflows, err := shuffle.GetAllWorkflowsByQuery(ctx, foundUser, 250, "")
if err != nil {
log.Printf("[ERROR] Failed getting backup workflows for org %s: %s", org.Id, err)
} else {
backupJob.Workflows = workflows
}
}
if org.SyncConfig.AppBackup && len(org.Users) > 0 {
foundUser.ActiveOrg.Id = org.Id
apps, err := shuffle.GetPrioritizedApps(ctx, foundUser)
if err != nil {
log.Printf("[ERROR] Failed getting backup apps for org %s: %s", org.Id, err)
} else {
parsedApps := []shuffle.WorkflowApp{}
for _, app := range apps {
if len(app.Actions) == 0 {
continue
}
if !app.Generated {
continue
}
parsedApps = append(parsedApps, app)
}
backupJob.Apps = parsedApps
}
}
// Send stats once every 10 times or so..?
// For now, just send every time
info, err := shuffle.GetOrgStatistics(ctx, org.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err)
} else {
backupJob.Stats = *info
}
}
backupJobData, err := json.Marshal(backupJob)
if err != nil {
log.Printf("[ERROR] Failed marshalling backup job: %s", err)
backupJobData = []byte{}
}
cloudSyncRegionUrlCacheKey := fmt.Sprintf("org_cloudsync_region_url_%s", org.Id)
cloudSyncRegionUrlCached, err := shuffle.GetCache(ctx, cloudSyncRegionUrlCacheKey)
if err == nil {
if cachedBytes, ok := cloudSyncRegionUrlCached.([]byte); ok && len(cachedBytes) > 0 {
syncUrl = string(cachedBytes)
log.Printf("[DEBUG] Using cached cloud sync region url for org %s: %s", org.Id, syncUrl)
}
}
if len(syncUrl) == 0 || !strings.HasPrefix(syncUrl, "http") {
syncUrl = "https://shuffler.io"
}
syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync", syncUrl)
client := shuffle.GetExternalClient(syncUrl)
req, err := http.NewRequest(
"POST",
syncUrl,
bytes.NewBuffer(backupJobData),
)
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey))
newresp, err := client.Do(req)
if err != nil {
//log.Printf("Failed request in org sync: %s", err)
return err
}
defer newresp.Body.Close()
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed body read in job sync: %s", err)
return err
}
//log.Printf("Remote Data: %s", respBody)
err = remoteOrgJobController(org, respBody)
if err != nil {
//log.Printf("[ERROR] Failed cloud sync job controller run for '%s': %s", respBody, err)
return err
}
return nil
}
// /api/v1/mcp
// /api/v1/agent
// /api/v1/apps/{appid}/mcp
func runMCPAction(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
ctx := shuffle.GetContext(request)
parentExec := shuffle.WorkflowExecution{}
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
// Look for org_id query as app may be private
// No validation is done here, as it's just running the app
// to find a user
orgId := request.URL.Query().Get("org_id")
if len(orgId) > 0 {
user.ActiveOrg.Id = orgId
} else {
executionId := request.URL.Query().Get("execution_id")
authorization := request.URL.Query().Get("authorization")
if len(executionId) == 0 || len(authorization) == 0 {
log.Printf("[WARNING] Bad execution id/auth in single action validate (1): %#v, %#v. Continuing with the 'public' org id", executionId, authorization)
err := shuffle.ValidateRequestOverload(resp, request)
if err != nil {
log.Printf("[INFO] Request overload for IP %s in single action execution", shuffle.GetRequestIp(request))
resp.WriteHeader(429)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Too many requests. Please try again in 30 seconds."}`)))
return
}
user.Username = shuffle.GetRequestIp(request)
user.ActiveOrg.Name = shuffle.GetRequestIp(request)
user.ActiveOrg.Id = "public"
} else {
// Find the execution
exec, err := shuffle.GetWorkflowExecution(ctx, executionId)
if err != nil {
log.Printf("[WARNING] Bad execution id in single action validate (2): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Bad execution mapping (1)"}`))
return
}
if exec.Authorization != authorization {
log.Printf("[WARNING] Bad execution auth in single action validate (3): %#v, %#v", exec.Authorization, authorization)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Bad execution mapping (2)"}`))
return
}
parentExec = *exec
user.ActiveOrg.Id = exec.OrgId
if len(user.ActiveOrg.Id) == 0 {
user.ActiveOrg.Id = exec.ExecutionOrg
}
user.Username = fmt.Sprintf("org %s", user.ActiveOrg.Id)
}
}
if len(user.ActiveOrg.Id) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No org_id found to map back to"}`))
return
}
}
location := strings.Split(request.URL.Path, "/")
var fileId string
if location[1] == "api" {
if len(location) <= 3 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if len(location) == 4 {
fileId = location[3] // /api/v1/agent or /api/v1/mcp
} else {
fileId = location[4] // /api/v1/apps/{appid}/mcp
}
}
//log.Printf("[AUDIT] User Authentication failed in execute SINGLE action - CONTINUING ANYWAY: %s. Found OrgID: %#v", err, user.ActiveOrg.Id)
log.Printf("[AUDIT] User %s (%s) in org %s (%s) is running SINGLE App run for App ID '%s'", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, fileId)
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[INFO] Failed single execution POST body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
foundRequest := shuffle.MCPRequest{}
//func HandleAiAgentExecutionStart(execution WorkflowExecution, startNode Action, createNextActions bool) (Action, error) {
// Unmarshal it
err = json.Unmarshal(body, &foundRequest)
if err != nil {
log.Printf("[INFO] Failed single execution POST body unmarshal: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
foundEnvironment := "Shuffle"
if len(foundRequest.Params.Environment) > 0 {
foundEnvironment = foundRequest.Params.Environment
}
if len(foundRequest.Params.Input.Text) < 5 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Input text is required and must be at least 5 characters"}`))
return
}
var newAction shuffle.Action
if fileId == "agent" {
newAction = shuffle.Action{
Name: "agent",
AppName: "AI Agent",
AppID: "shuffle_agent",
AppVersion: "1.0.0",
Environment: foundEnvironment,
Parameters: []shuffle.WorkflowAppActionParameter{
{
Name: "app_name",
Value: "openai",
},
{
Name: "input",
Value: foundRequest.Params.Input.Text,
},
},
}
} else {
foundId := ""
if len(foundRequest.Params.ToolID) > 0 {
foundId = foundRequest.Params.ToolID
} else {
if len(foundRequest.Params.ToolName) == 32 {
foundId = foundRequest.Params.ToolName
} else {
foundApps, err := shuffle.FindWorkflowAppByName(ctx, foundRequest.Params.ToolName)
if err != nil || len(foundApps) == 0 {
log.Printf("[INFO] Failed to find app by name '%s' in single execution: %s", foundRequest.Params.ToolName, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Valid param.tool_id (app ID) is required"}`))
return
}
for _, app := range foundApps {
if app.Name == foundRequest.Params.ToolName {
foundId = app.ID
break
}
}
}
}
app, err := shuffle.GetApp(ctx, foundId, shuffle.User{}, false)
if err != nil {
log.Printf("[INFO] Failed to find app by id '%s' in single execution: %s", foundId, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if !app.Public {
if user.Id == app.Owner || user.ActiveOrg.Id == app.ReferenceOrg || shuffle.ArrayContains(app.Contributors, user.Id) {
log.Printf("[AUDIT] Support & Admin user %s (%s) got access to app %s (MCP)", user.Username, user.Id, app.ID)
} else if user.Role == "admin" && app.Owner == "" {
log.Printf("[AUDIT] Any admin can GET %s (%s), since it doesn't have an owner (GET - MCP).", app.Name, app.ID)
} else {
log.Printf("[AUDIT] User %s (%s) in org %s (%s) was denied access to app %s (MCP)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, app.ID)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
} else {
log.Printf("[AUDIT] User %s (%s) in org %s (%s) got access to public app %s (MCP)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id, app.ID)
}
// Check permissions
parsedName := strings.ToLower(strings.ReplaceAll(app.Name, " ", "_"))
parsedApp := fmt.Sprintf("app:%s:%s", app.ID, parsedName)
// Run the action
newAction = shuffle.Action{
Name: "agent",
AppName: "AI Agent",
AppID: "shuffle_agent",
AppVersion: "1.0.0",
Environment: foundEnvironment,
Parameters: []shuffle.WorkflowAppActionParameter{
{
Name: "app_name",
Value: "openai",
},
{
Name: "input",
Value: foundRequest.Params.Input.Text,
},
{
Name: "app_name",
Value: parsedApp,
},
},
}
}
marshalledAction, err := json.Marshal(newAction)
if err != nil {
log.Printf("[ERROR] Failed to marshal single action body: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
// Special handling for agent calls
if fileId == "agent" {
if len(parentExec.ExecutionId) > 0 {
targetActionId := request.URL.Query().Get("action_id")
agentNodeFound := false
var agentNode shuffle.Action
for i, action := range parentExec.Workflow.Actions {
if len(targetActionId) > 0 && action.ID != targetActionId {
continue
}
if len(targetActionId) == 0 && action.AppName != "AI Agent" {
continue
}
if len(foundRequest.Params.Input.Text) > 0 {
for j, param := range parentExec.Workflow.Actions[i].Parameters {
if param.Name == "input" {
parentExec.Workflow.Actions[i].Parameters[j].Value = foundRequest.Params.Input.Text
break
}
}
}
agentNode = parentExec.Workflow.Actions[i]
agentNodeFound = true
log.Printf("[DEBUG][%s] AI Agent: Running AI Agent node '%s' (ID: %s)", parentExec.ExecutionId, agentNode.Label, agentNode.ID)
break
}
if !agentNodeFound {
log.Printf("[ERROR][%s] No AI Agent node found in parent workflow (Target ID: %s)", parentExec.ExecutionId, targetActionId)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No AI Agent node found in parent workflow matching request"}`)))
return
}
callerName := "deployAppShuffleOnprem"
go shuffle.HandleAiAgentExecutionStart(parentExec, agentNode, false, callerName)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s", "mode": "hybrid"}`, parentExec.ExecutionId, parentExec.Authorization)))
return
} else {
//standalone mode
workflowExecution, err := shuffle.PrepareSingleAction(ctx, request, user, "agent_starter", marshalledAction, false, "")
if err != nil {
log.Printf("[ERROR] Failed to prepare standalone agent execution: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to create agent execution"}`))
return
}
log.Printf("[INFO] Standalone agent execution created: %s", workflowExecution.ExecutionId)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s", "mode": "standalone"}`, workflowExecution.ExecutionId, workflowExecution.Authorization)))
return
}
}
// For non-agent calls (MCP or other apps): continue with existing flow
workflowExecution, err := shuffle.PrepareSingleAction(ctx, request, user, "agent", marshalledAction, false, "")
if fileId == "agent_starter" {
log.Printf("[INFO] Returning early for agent_starter single action execution: %s", workflowExecution.ExecutionId)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization)))
return
}
debugUrl := fmt.Sprintf("/workflows/%s?execution_id=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
resp.Header().Add("X-Debug-Url", debugUrl)
if err != nil {
returndata := shuffle.ResultChecker{
Success: false,
Reason: fmt.Sprintf("%s", err),
}
// Special handler for decision reruns~
if strings.Contains(err.Error(), "Successfully") {
returndata.Success = true
resp.WriteHeader(200)
} else {
log.Printf("[INFO] Failed workflowrequest POST read in single action (4): %s", err)
resp.WriteHeader(400)
}
respBytes, err := json.Marshal(returndata)
if err != nil {
resp.Write([]byte(`{"success": false}`))
return
}
resp.Write(respBytes)
return
}
foundEnv := ""
params := []string{}
for _, action := range workflowExecution.Workflow.Actions {
for _, param := range action.Parameters {
params = append(params, param.Name)
}
if len(action.Environment) > 0 {
foundEnv = action.Environment
break
}
}
go shuffle.IncrementCache(ctx, workflowExecution.OrgId, "workflow_executions")
executionRequest := shuffle.ExecutionRequest{
ExecutionId: workflowExecution.ExecutionId,
WorkflowId: workflowExecution.Workflow.ID,
Authorization: workflowExecution.Authorization,
Environments: []string{foundEnv},
}
parsedEnv := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(foundEnv, " ", "-"), "_", "-")), workflowExecution.ExecutionOrg)
// Check if environment is distributed from parent org
if len(workflowExecution.ExecutionOrg) > 0 {
environments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("[ERROR] Failed getting environments for org %s in single action. May fail to verify env.: %s", workflowExecution.ExecutionOrg, err)
} else {
for _, env := range environments {
if env.Archived {
continue
}
if env.Name != foundEnv {
continue
}
if env.OrgId != workflowExecution.ExecutionOrg && len(env.OrgId) > 0 {
if debug {
log.Printf("[DEBUG][%s] Found suborg environment %s for org %s in single action. Re-mapping it to org-id %s", workflowExecution.ExecutionId, env.Name, env.OrgId, env.OrgId)
}
parsedEnv = fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(foundEnv, " ", "-"), "_", "-")), env.OrgId)
break
}
}
}
}
log.Printf("[INFO][%s] Adding new single-action job to env queue (4 - MCP): %s", workflowExecution.ExecutionId, parsedEnv)
err = shuffle.SetWorkflowQueue(ctx, executionRequest, parsedEnv)
if err != nil {
log.Printf("[WARNING][%s] Failed adding %s to db (single action queue): %s", workflowExecution.ExecutionId, parsedEnv, err)
}
actionId := ""
if len(workflowExecution.Workflow.Actions) == 1 {
actionId = workflowExecution.Workflow.Actions[0].ID
}
mappedResponse := shuffle.MCPResponse{
Jsonrpc: foundRequest.Jsonrpc,
ID: foundRequest.ID,
}
singleResult := shuffle.HandleRetValidation(ctx, workflowExecution, 1, 45, actionId)
agentOutput := shuffle.AgentOutput{}
err = json.Unmarshal([]byte(singleResult.Result), &agentOutput)
if err == nil && len(agentOutput.Output) > 0 {
log.Printf("[INFO] Returning agent output in MCP response: %s", agentOutput.Output)
marshalledAgentOutput, err := json.Marshal(agentOutput)
if err != nil {
log.Printf("[ERROR] Failed to marshal agent output in MCP response: %s", err)
} else {
marshalledOutput := map[string]interface{}{}
err = json.Unmarshal(marshalledAgentOutput, &marshalledOutput)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal agent output in MCP response: %s", err)
}
marshalledOutput["message"] = agentOutput.Output
mappedResponse.Result = marshalledOutput
}
} else {
marshalledSingleResult, err := json.Marshal(singleResult)
if err != nil {
log.Printf("[ERROR] Failed to marshal single result in MCP response: %s", err)
}
marshalledResult := map[string]interface{}{}
err = json.Unmarshal(marshalledSingleResult, &marshalledResult)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal single result in MCP response: %s", err)
}
}
marshalledMappedResponse, err := json.Marshal(mappedResponse)
if err != nil {
log.Printf("[ERROR] Failed to marshal mapped response in MCP response: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(marshalledMappedResponse)
}
func runInitCloudSetup() {
action := shuffle.CloudSyncJob{
Type: "setup",
Action: "init",
OrgId: "",
PrimaryItemId: "",
}
err := executeCloudAction(action, "")
if err != nil {
log.Printf("[INFO] Failed initial setup: %s", err)
} else {
log.Printf("[INFO] Finished initial cloudsync setup!")
}
}
func runInitEs(ctx context.Context) {
log.Printf("[DEBUG] Starting INIT setup for Elasticsearch/Opensearch")
httpProxy := os.Getenv("HTTP_PROXY")
if len(httpProxy) > 0 {
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
httpsProxy := os.Getenv("HTTPS_PROXY")
if len(httpsProxy) > 0 {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
defaultEnv := os.Getenv("ENVIRONMENT_NAME")
if len(defaultEnv) == 0 {
defaultEnv = "Shuffle"
log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv)
}
log.Printf("[DEBUG] Getting organizations for Elasticsearch/Opensearch")
activeOrgs, err := shuffle.GetAllOrgs(ctx)
log.Printf("[DEBUG] Got %d organizations to look into. If this is 0, we wait 10 more seconds until DB is ready and try again.", len(activeOrgs))
if len(activeOrgs) == 0 {
time.Sleep(10 * time.Second)
activeOrgs, err = shuffle.GetAllOrgs(ctx)
}
setUsers := false
_ = setUsers
if err != nil {
if fmt.Sprintf("%s", err) == "EOF" || strings.Contains(fmt.Sprintf("%s", err), "bad status") {
time.Sleep(10 * time.Second)
runInitEs(ctx)
return
}
log.Printf("[DEBUG] Error getting organizations: %s", err)
runInitCloudSetup()
} else {
// Add all users to it
if len(activeOrgs) == 1 {
setUsers = true
} else if len(activeOrgs) == 0 {
log.Printf(`[DEBUG] No orgs. Setting NEW org "default"`)
runInitCloudSetup()
//orgSetupName := "default"
//orgId := uuid.NewV4().String()
//newOrg := shuffle.Org{
// Name: orgSetupName,
// Id: orgId,
// Org: orgSetupName,
// Users: []shuffle.User{},
// Roles: []string{"admin", "user"},
// CloudSync: false,
//}
//err = shuffle.SetOrg(ctx, newOrg, orgId)
//if err != nil {
// log.Printf("Failed setting organization: %s", err)
//} else {
// log.Printf("Successfully created the default org!")
// setUsers = true
// item := shuffle.Environment{
// Name: defaultEnv,
// Type: "onprem",
// OrgId: orgId,
// Default: true,
// Id: uuid.NewV4().String(),
// }
// err = shuffle.SetEnvironment(ctx, &item)
// if err != nil {
// log.Printf("[WARNING] Failed setting up new environment for new org")
// }
//}
} else {
log.Printf("[DEBUG] Found %d org(s) in total.", len(activeOrgs))
if len(activeOrgs) == 1 {
if len(activeOrgs[0].Users) == 0 {
log.Printf("[ERROR] Main Org doesn't have any user. Creating.")
users, err := shuffle.GetAllUsers(ctx)
if err != nil && len(users) == 0 {
log.Printf("Failed getting users in org fix")
} else {
// Remapping everyone to admin. This should never happen.
for _, user := range users {
user.ActiveOrg = shuffle.OrgMini{
Id: activeOrgs[0].Id,
Name: activeOrgs[0].Name,
Role: "admin",
}
activeOrgs[0].Users = append(activeOrgs[0].Users, user)
}
err = shuffle.SetOrg(ctx, activeOrgs[0], activeOrgs[0].Id)
if err != nil {
log.Printf("Failed setting org: %s", err)
} else {
log.Printf("Successfully updated org to have users!")
}
}
}
}
}
}
log.Printf("[INFO] Waiting 30 seconds during init to make sure the opensearch instance is up and running with security features enabled")
time.Sleep(30 * time.Second)
shuffle.InitOpensearchIndexes()
// FIXME: This should ONLY run on one backend instance. This may cause interference.
schedules, err := shuffle.GetAllSchedules(ctx, "ALL")
if err != nil {
log.Printf("[WARNING] Failed getting schedules during service init: %s", err)
} else {
log.Printf("[INFO] Setting up %d schedule(s)", len(schedules))
url := &url.URL{}
job := func(schedule shuffle.ScheduleOld) func() {
return func() {
log.Printf("[INFO] Running schedule %s with interval %d.", schedule.Id, schedule.Seconds)
request := &http.Request{
URL: url,
Method: "POST",
Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)),
}
orgId := ""
if len(activeOrgs) > 0 {
orgId = activeOrgs[0].Id
}
if len(schedule.Org) == 36 {
orgId = schedule.Org
}
_, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request, orgId)
if err != nil {
log.Printf("[WARNING] Failed to execute %s: %s", schedule.WorkflowId, err)
}
}
}
for _, schedule := range schedules {
if strings.ToLower(schedule.Environment) == "cloud" {
log.Printf("[DEBUG] Skipping cloud schedule")
continue
}
// FIXME: Add a randomized timer to avoid all schedules running at the same time
// Many are at 5 minutes / 1 hour. The point is to spread these out
// a bit instead of all of them starting at the exact same time
//log.Printf("Schedule: %#v", schedule)
//log.Printf("Schedule time: every %d seconds", schedule.Seconds)
if schedule.Seconds == 0 && len(schedule.Frequency) > 0 {
cronJob, err := CronScheduler.Cron(schedule.Frequency).Do(job(schedule))
if err != nil {
log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err)
} else {
log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId)
}
cronJobs[schedule.Id] = cronJob
} else {
jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job(schedule))
if err != nil {
log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", schedule.WorkflowId, err)
} else {
log.Printf("[DEBUG] Successfully started schedule for workflow %s", schedule.WorkflowId)
}
scheduledJobs[schedule.Id] = jobret
}
}
}
parsedApikey := ""
users, err := shuffle.GetAllUsers(ctx)
if len(users) == 0 {
log.Printf("[INFO] Trying to set up user based on environments SHUFFLE_DEFAULT_USERNAME & SHUFFLE_DEFAULT_PASSWORD")
username := os.Getenv("SHUFFLE_DEFAULT_USERNAME")
password := os.Getenv("SHUFFLE_DEFAULT_PASSWORD")
if len(username) == 0 || len(password) == 0 || len(activeOrgs) > 0 {
log.Printf("[DEBUG] SHUFFLE_DEFAULT_USERNAME and SHUFFLE_DEFAULT_PASSWORD not defined as environments. Running without default user.")
} else {
apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY")
if len(parsedApikey) == 0 {
parsedApikey = apikey
}
log.Printf("[DEBUG] Creating org for default user %s", username)
orgId := uuid.NewV4().String()
orgSetupName := "default"
tmpOrg := shuffle.OrgMini{
Name: orgSetupName,
Id: orgId,
}
err = createNewUser(username, password, "admin", apikey, tmpOrg)
if err != nil {
log.Printf("[ERROR] Failed to create default user %s: %s", username, err)
} else {
log.Printf("[INFO] Successfully created user %s", username)
}
user, err := shuffle.GetUser(ctx, username)
newOrg := shuffle.Org{
Name: orgSetupName,
Id: orgId,
Org: orgSetupName,
Users: []shuffle.User{*user},
Roles: []string{"admin", "user"},
CloudSync: false,
}
err = shuffle.SetOrg(ctx, newOrg, newOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed setting organization when creating original user: %s", err)
} else {
log.Printf("[DEBUG] Successfully created the default org with id %s!", orgId)
item := shuffle.Environment{
Name: defaultEnv,
Type: "onprem",
OrgId: orgId,
Default: true,
Id: uuid.NewV4().String(),
}
err = shuffle.SetEnvironment(ctx, &item)
if err != nil {
log.Printf("[WARNING] Failed setting up new environment")
}
}
}
} else {
for _, user := range users {
if user.Role == "admin" && len(user.ApiKey) > 0 {
parsedApikey = user.ApiKey
log.Printf("[DEBUG] Using apikey of %s (%s) for cleanup", user.Username, user.Id)
break
}
}
}
log.Printf("[INFO] Starting cloud schedules for orgs if enabled!")
type requestStruct struct {
ApiKey string `json:"api_key"`
}
for _, org := range activeOrgs {
if len(org.Id) == 0 {
log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name)
continue
}
if !org.CloudSync {
log.Printf("[INFO] Skipping org syncCheck for '%s' because sync isn't set (1).", org.Id)
continue
}
//interval := int(org.SyncConfig.Interval)
interval := 30
if interval == 0 {
log.Printf("[WARNING] Skipping org %s because sync isn't set (0).", org.Id)
continue
}
log.Printf("[DEBUG] Should start cloud schedule for org %s (%s)", org.Name, org.Id)
job := func() {
err := remoteOrgJobHandler(org, interval)
if err != nil {
log.Printf("[ERROR] Failed request with remote org sync for org %s (2): %s", org.Id, err)
}
}
jobret, err := newscheduler.Every(int(interval)).Seconds().NotImmediately().Run(job)
if err != nil {
log.Printf("[ERROR] Failed to schedule org: %s", err)
} else {
log.Printf("[INFO] Started sync on interval %d for org %s (%s)", interval, org.Name, org.Id)
scheduledOrgs[org.Id] = jobret
}
}
forceUpdateEnv := os.Getenv("SHUFFLE_APP_FORCE_UPDATE")
forceUpdate := false
if len(forceUpdateEnv) > 0 && forceUpdateEnv == "true" {
log.Printf("Forcing to rebuild apps")
forceUpdate = true
}
// FIXME: Have this for all envs in all orgs (loop and find).
if len(parsedApikey) == 0 {
log.Printf("[WARNING] No apikey found for cleanup. Skipping cleanup schedule.")
} else {
cleanupSchedule := 300
if len(os.Getenv("SHUFFLE_RERUN_SCHEDULE")) > 0 {
newfrequency, err := strconv.Atoi(os.Getenv("SHUFFLE_RERUN_SCHEDULE"))
if err == nil {
cleanupSchedule = newfrequency
if cleanupSchedule < 300 {
log.Printf("[WARNING] A Cleanupschedule of less than 300 seconds won't help.")
cleanupSchedule = 300
}
}
}
environments := []string{defaultEnv}
// Comma separated list of RERUN environments
if len(os.Getenv("SHUFFLE_RERUN_ENVIRONMENTS")) > 0 {
foundenv := strings.Split(os.Getenv("SHUFFLE_RERUN_ENVIRONMENTS"), ",")
for i, env := range foundenv {
if len(env) == 0 {
continue
}
environments[i] = strings.TrimSpace(env)
if !shuffle.ArrayContains(environments, env) {
environments = append(foundenv, env)
}
}
}
log.Printf("[DEBUG] Starting schedule setup for execution cleanup every %d seconds. Running first immediately. Environments: %#v", cleanupSchedule, environments)
cleanupJob := func() func() {
return func() {
//log.Printf("[INFO] Running schedule for cleaning up or re-running unfinished workflows in %d environments.", len(environments))
backendPort := os.Getenv("BACKEND_PORT")
if backendPort == "" {
backendPort = "5001"
}
for _, environment := range environments {
// Allowed without PROXY management as it's localhost
// client := shuffle.GetExternalClient(syncUrl)
httpClient := &http.Client{}
url := fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/stop", backendPort, environment)
req, err := http.NewRequest(
"GET",
url,
nil,
)
// FIXME: This will stop working of the user rotates their key lol
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, parsedApikey))
if err != nil {
log.Printf("[ERROR] Failed CREATING environment request for %s: %s", environment, err)
continue
}
newresp, err := httpClient.Do(req)
if err != nil {
log.Printf("[ERROR] Failed running environment request %s: %s", environment, err)
continue
}
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed setting respbody %s for execution stop. Status: %d", err, newresp.StatusCode)
continue
}
if newresp.StatusCode != 200 {
if !strings.Contains(string(respBody), "is active") {
log.Printf("[WARNING] Failed stopping runs in environment %s. Status code: %d. Body: %s", environment, newresp.StatusCode, string(respBody))
}
continue
}
url = fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/rerun", backendPort, environment)
req, err = http.NewRequest(
"GET",
url,
nil,
)
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, parsedApikey))
if err != nil {
log.Printf("[ERROR] Failed CREATING environment request to rerun for %s: %s", environment, err)
continue
}
newresp, err = httpClient.Do(req)
if err != nil {
log.Printf("[ERROR] Failed running environment request to rerun for %s: %s", environment, err)
continue
}
if newresp.StatusCode != 200 {
log.Printf("[WARNING] Failed rerunning environment %s. Status code: %d", environment, newresp.StatusCode)
}
//respBody, err := ioutil.ReadAll(newresp.Body)
//if err != nil {
// log.Printf("[ERROR] Failed setting respbody %s", err)
// continue
//}
//log.Printf("[DEBUG] Ran workflow RERUN request for %s with the response. Body: %s", environment, string(respBody))
}
}
}
jobret, err := newscheduler.Every(cleanupSchedule).Seconds().Run(cleanupJob())
if err != nil {
log.Printf("[ERROR] Failed to schedule Cleanup: %s", err)
} else {
_ = jobret
}
}
// Getting apps to see if we should initialize a test
workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 1000, 0)
log.Printf("[INFO] Getting and validating workflowapps. Got %d with err %#v", len(workflowapps), err)
if err != nil && len(workflowapps) == 0 {
log.Printf("[WARNING] Failed getting apps (runInit): %s", err)
} else if err == nil && len(workflowapps) < 10 {
log.Printf("[DEBUG] Downloading default apps as %d were found", len(workflowapps))
fs := memfs.New()
storer := memory.NewStorage()
url := os.Getenv("SHUFFLE_APP_DOWNLOAD_LOCATION")
if len(url) == 0 {
log.Printf("[INFO] Skipping download of apps since no URL is set. Default would be https://github.com/shuffle/python-apps")
url = "https://github.com/shuffle/python-apps"
//url = ""
//return
}
username := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_USERNAME")
password := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_PASSWORD")
cloneOptions := &git.CloneOptions{
URL: url,
}
if len(username) > 0 && len(password) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: username,
Password: password,
}
}
cloneOptions = shuffle.CheckGitProxy(cloneOptions)
branch := os.Getenv("SHUFFLE_DOWNLOAD_AUTH_BRANCH")
if len(branch) > 0 && branch != "master" && branch != "main" {
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
}
log.Printf("[DEBUG] Getting apps from url '%s'", url)
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("[ERROR] Failed loading repo into memory (init): %s", err)
}
dir, err := fs.ReadDir("")
if err != nil {
log.Printf("[WARNING] Failed reading folder (init): %s", err)
}
_ = r
//iterateAppGithubFolders(fs, dir, "", "testing")
_, _, err = IterateAppGithubFolders(ctx, fs, dir, "", "", forceUpdate, true)
if err != nil {
log.Printf("[WARNING] Error from app load in init: %s", err)
}
//_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
// Hotloads locally
location := os.Getenv("SHUFFLE_APP_HOTLOAD_FOLDER")
if len(location) == 0 {
location = "./shuffle-apps"
}
if len(location) != 0 {
handleAppHotload(ctx, location, false)
}
} else {
log.Printf("[DEBUG] Skipping download of default apps as %d were found", len(workflowapps))
}
if os.Getenv("SHUFFLE_HEALTHCHECK_DISABLED") != "true" {
healthcheckInterval := 60
log.Printf("[INFO] Starting healthcheck job every %d minute. Stats available on /api/v1/health/stats, and dashboard on /health. Disable with SHUFFLE_HEALTHCHECK_DISABLED=true", healthcheckInterval)
job := func() {
// Prepare a fake http.responsewriter
resp := httptest.NewRecorder()
request := http.Request{}
// Add the "force=true" query to the fake request
request.URL, err = url.Parse("/api/v1/health?force=true")
if err != nil {
log.Printf("[ERROR] Failed to parse test url for healthstats: %s", err)
}
shuffle.RunOpsHealthCheck(resp, &request)
}
_, err := newscheduler.Every(int(healthcheckInterval)).Minutes().Run(job)
if err != nil {
log.Printf("[ERROR] Failed to schedule healthcheck: %s", err)
} else {
log.Printf("[DEBUG] Successfully started healthcheck interval of %d minutes", healthcheckInterval)
}
}
// Self-cleaning
go func() {
cursor := ""
cnt := 0
newCtx := context.Background()
for _, org := range activeOrgs {
if len(org.Id) == 0 {
log.Printf("[DEBUG] No ID found for org with name '%s'. Why was it made?", org.Name)
continue
}
log.Printf("[INFO] Starting self-cleanup of cache keys for org %s", org.Id)
for {
keys, newCursor, err := shuffle.GetAllCacheKeys(newCtx, org.Id, "", 1000, cursor)
if err != nil {
//log.Printf("[ERROR] Failed getting all cache keys for cleanup: %s", err)
break
}
if newCursor == cursor || len(newCursor) == 0 {
break
}
if len(keys) == 0 {
break
}
cursor = newCursor
cnt += 1
if cnt > 10 {
break
}
}
log.Printf("[INFO] Finished self-cleanup of cache keys for org %s", org.Id)
}
}()
log.Printf("[INFO] Finished INIT (ES)")
}
func handleVerifyCloudsync(orgId string) (shuffle.SyncFeatures, error) {
ctx := context.Background()
org, err := shuffle.GetOrg(ctx, orgId)
if err != nil {
return shuffle.SyncFeatures{}, err
}
//r.HandleFunc("/api/v1/getorgs", handleGetOrgs).Methods("GET", "OPTIONS")
syncURL := fmt.Sprintf("%s/api/v1/cloud/sync/get_access", syncUrl)
client := shuffle.GetExternalClient(syncURL)
req, err := http.NewRequest(
"GET",
syncURL,
nil,
)
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey))
newresp, err := client.Do(req)
if err != nil {
return shuffle.SyncFeatures{}, err
}
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
return shuffle.SyncFeatures{}, err
}
responseData := retStruct{}
err = json.Unmarshal(respBody, &responseData)
if err != nil {
return shuffle.SyncFeatures{}, err
}
if newresp.StatusCode != 200 {
return shuffle.SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode))
}
if !responseData.Success {
return shuffle.SyncFeatures{}, errors.New(responseData.Reason)
}
return responseData.SyncFeatures, nil
}
// Actually stops syncing with cloud for an org.
// Disables potential schedules, removes environments, breaks workflows etc.
func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) {
if len(org.SyncConfig.Apikey) == 0 {
return &org, errors.New(fmt.Sprintf("Couldn't find any sync key to disable org %s", org.Id))
}
log.Printf("[INFO] Should run cloud sync disable for org %s with URL %s", org.Id, syncUrl)
client := shuffle.GetExternalClient(syncUrl)
req, err := http.NewRequest(
"DELETE",
syncUrl,
nil,
)
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey))
newresp, err := client.Do(req)
if err != nil {
return &org, err
}
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
return &org, err
}
log.Printf("[INFO] Remote disable ret: %s", string(respBody))
responseData := retStruct{}
err = json.Unmarshal(respBody, &responseData)
if err != nil {
return &org, err
}
// FIXME: If it says bad API-key, stop cloud sync for the Org
if newresp.StatusCode != 200 {
return &org, errors.New(fmt.Sprintf("Got status code %d when disabling org remotely. Expected 200. Contact support.", newresp.StatusCode))
}
if !responseData.Success {
//log.Printf("Success reason: %s", responseData.Reason)
return &org, errors.New(responseData.Reason)
}
log.Printf("[INFO] Everything is success. Should disable org sync for %s", org.Id)
ctx := context.Background()
org.CloudSync = false
org.SyncFeatures = shuffle.SyncFeatures{}
org.SyncConfig = shuffle.SyncConfig{}
org.Subscriptions = []shuffle.PaymentSubscription{}
err = shuffle.SetOrg(ctx, org, org.Id)
if err != nil {
newerror := fmt.Sprintf("[WARNING] ERROR: Failed updating even though there was success: %s", err)
log.Printf(newerror)
return &org, errors.New(newerror)
}
environments, err := shuffle.GetEnvironments(ctx, org.Id)
if err != nil {
log.Printf("[WARNING] Failed getting envs in stop sync: %s", err)
return &org, err
}
// Don't disable, this will be deleted entirely
for _, environment := range environments {
if environment.Type == "cloud" {
environment.Name = "Cloud"
environment.Archived = true
err = shuffle.SetEnvironment(ctx, &environment)
if err == nil {
log.Printf("[INFO] Updated cloud environment %s", environment.Name)
} else {
log.Printf("[INFO] Failed to update cloud environment %s", environment.Name)
}
}
}
// FIXME: This doesn't work?
if value, exists := scheduledOrgs[org.Id]; exists {
// Looks like this does the trick? Hurr
log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id)
value.Lock()
}
return &org, nil
}
// INFO: https://docs.google.com/drawings/d/1JJebpPeEVEbmH_qsAC6zf9Noygp7PytvesrkhE19QrY/edit
/*
This is here to both enable and disable cloud sync features for an organization
*/
func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
user, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in cloud setup: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("Not admin.")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type ReturnData struct {
Apikey string `datastore:"apikey"`
Organization shuffle.Org `datastore:"organization"`
Disable bool `datastore:"disable"`
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("Failed unmarshalling test: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := context.Background()
org, err := shuffle.GetOrg(ctx, tmpData.Organization.Id)
if err != nil {
log.Printf("Organization doesn't exist: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: Check if user is admin of this org
//log.Printf("Checking org %s", org.Name)
userFound := false
admin := false
for _, inneruser := range org.Users {
if inneruser.Id == user.Id {
userFound = true
//log.Printf("[INFO] Role: %s", inneruser.Role)
if inneruser.Role == "admin" {
admin = true
}
break
}
}
if !userFound {
log.Printf("User %s doesn't exist in organization %s", user.Id, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME: Enable admin check in org for sync setup and conf.
_ = admin
//if !admin {
// log.Printf("User %s isn't admin hence can't set up sync for org %s", user.Id, org.Id)
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false}`))
// return
//}
//log.Printf("Apidata: %s", tmpData.Apikey)
// FIXME: Path
// just in case setup/stop URL is overwritten by region url
if syncUrl != "https://shuffler.io" || syncUrl != "http://localhost:5002" {
syncUrl = "https://shuffler.io"
}
apiPath := "/api/v1/cloud/sync/setup"
if tmpData.Disable {
if !org.CloudSync {
log.Printf("[WARNING] Org %s isn't syncing. Can't stop.", org.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Skipped cloud sync setup. Already syncing."}`)))
return
}
log.Printf("[INFO] Should disable sync for org %s", org.Id)
apiPath := "/api/v1/cloud/sync/stop"
syncPath := fmt.Sprintf("%s%s", syncUrl, apiPath)
_, err = handleStopCloudSync(syncPath, *org)
if err != nil {
ret := shuffle.ResultChecker{
Success: false,
Reason: fmt.Sprintf("%s", err),
}
resp.WriteHeader(401)
b, err := json.Marshal(ret)
if err != nil {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
resp.Write(b)
} else {
// Delete cache for the sync features
cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id)
shuffle.DeleteCache(ctx, cacheKey)
subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id)
shuffle.DeleteCache(ctx, subscriptionCacheKey)
licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id)
shuffle.DeleteCache(ctx, licenseCacheKey)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully disabled cloud sync for org."}`)))
}
return
}
// Everything below here is to SET UP CLOUD SYNC.
// If you want to disable cloud sync, see previous section.
if org.CloudSync {
log.Printf("[WARNING] Org %s is already syncing. Skip", org.Id)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Your org is already syncing. Nothing to set up."}`)))
return
}
syncPath := fmt.Sprintf("%s%s", syncUrl, apiPath)
type requestStruct struct {
ApiKey string `json:"api_key"`
}
requestData := requestStruct{
ApiKey: tmpData.Apikey,
}
b, err := json.Marshal(requestData)
if err != nil {
log.Printf("[ERROR] Failed marshaling api key data: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed cloud sync: %s"}`, err)))
return
}
req, err := http.NewRequest(
"POST",
syncPath,
bytes.NewBuffer(b),
)
client := shuffle.GetExternalClient(syncPath)
newresp, err := client.Do(req)
if err != nil {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed cloud sync: %s. Contact support."}`, err)))
//setBadMemcache(ctx, docPath)
return
}
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse sync data. Contact support."}`)))
return
}
responseData := retStruct{}
err = json.Unmarshal(respBody, &responseData)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed handling cloud data"}`)))
return
}
if newresp.StatusCode != 200 {
resp.WriteHeader(401)
resp.Write(respBody)
return
}
if !responseData.Success {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, responseData.Reason)))
return
}
// FIXME:
// 1. Set cloudsync for org to be active
// 2. Add iterative sync schedule for interval seconds
// 3. Add another environment for the org's users
org.CloudSync = true
// set cache here for 30 min
cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id)
featuresBytes, err := json.Marshal(responseData.SyncFeatures)
if err != nil {
log.Printf("[ERROR] Failed to marshal SyncFeatures for cache: %s", err)
} else {
shuffle.SetCache(ctx, cacheKey, featuresBytes, 1800)
}
subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id)
subscriptionsBytes, err := json.Marshal(responseData.Subscriptions)
if err != nil {
log.Printf("[ERROR] Failed to marshal Subscriptions for cache: %s", err)
} else {
shuffle.SetCache(ctx, subscriptionCacheKey, subscriptionsBytes, 1800)
}
licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id)
licensedBytes, err := json.Marshal(responseData.Licensed)
if err != nil {
log.Printf("[ERROR] Failed to marshal Licensed for cache: %s", err)
} else {
shuffle.SetCache(ctx, licenseCacheKey, licensedBytes, 1800)
}
if len(responseData.CloudSyncUrl) > 0 {
cloudSyncRegionUrlCacheKey := fmt.Sprintf("org_cloudsync_region_url_%s", org.Id)
shuffle.SetCache(ctx, cloudSyncRegionUrlCacheKey, []byte(responseData.CloudSyncUrl), 1800)
}
org.SyncConfig = shuffle.SyncConfig{
Apikey: responseData.SessionKey,
Interval: responseData.IntervalSeconds,
WorkflowBackup: true,
AppBackup: true,
}
interval := int(responseData.IntervalSeconds)
log.Printf("[INFO] Starting cloud sync on interval %d", interval)
job := func() {
err := remoteOrgJobHandler(*org, interval)
if err != nil {
log.Printf("[ERROR] Failed request with remote org sync (1): %s", err)
}
}
jobret, err := newscheduler.Every(int(interval)).Seconds().NotImmediately().Run(job)
if err != nil {
log.Printf("[ERROR] Failed to schedule org: %s", err)
} else {
log.Printf("[INFO] Started sync on interval %d for org %s", interval, org.Name)
scheduledOrgs[org.Id] = jobret
}
// ONLY checked added if workflows are allow huh
if org.SyncFeatures.Workflows.Active {
log.Printf("[INFO] Should activate cloud workflows for org %s!", org.Id)
// 1. Find environment
// 2. If cloud env found, enable it (un-archive)
// 3. If it doesn't create it
environments, err := shuffle.GetEnvironments(ctx, org.Id)
log.Printf("GETTING ENVS: %#v", environments)
if err == nil {
// Don't disable, this will be deleted entirely
found := false
for _, environment := range environments {
if environment.Type == "cloud" {
environment.Name = "Cloud"
environment.Archived = false
err = shuffle.SetEnvironment(ctx, &environment)
if err == nil {
log.Printf("[INFO] Re-added cloud environment %s", environment.Name)
} else {
log.Printf("[INFO] Failed to re-enable cloud environment %s", environment.Name)
}
found = true
break
}
}
if !found {
log.Printf("[INFO] Env for cloud not found. Should add it!")
newEnv := shuffle.Environment{
Name: "Cloud",
Type: "cloud",
Archived: false,
Registered: true,
Default: false,
OrgId: org.Id,
Id: uuid.NewV4().String(),
}
err = shuffle.SetEnvironment(ctx, &newEnv)
if err != nil {
log.Printf("Failed setting up NEW org environment for org %s: %s", org.Id, err)
} else {
log.Printf("Successfully added new environment for org %s", org.Id)
}
}
} else {
log.Printf("Failed setting org environment, because none were found: %s", err)
}
}
err = shuffle.SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("ERROR: Failed updating org even though there was success: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting up org after sync success. Contact support."}`)))
return
}
if responseData.IntervalSeconds > 0 {
// FIXME:
log.Printf("[INFO] Should set up interval for %d with session key %s for org %s", responseData.IntervalSeconds, responseData.SessionKey, org.Name)
}
resp.WriteHeader(200)
resp.Write(respBody)
}
func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
user, userErr := shuffle.HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in make workflow public: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access publish workflow: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
ctx := context.Background()
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`))
return
}
workflow, err := shuffle.GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Workflow %s doesn't exist in app publish. User: %s", fileId, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// CHECK orgs of user, or if user is owner
// FIXME - add org check too, and not just owner
// Check workflow.Sharing == private / public / org too
if user.Id != workflow.Owner || len(user.Id) == 0 {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (public)", user.Username, workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (public)", user.Username, workflow.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if !workflow.IsValid || !workflow.PreviouslySaved {
log.Printf("[INFO] Failed uploading workflow because it's invalid or not saved")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid workflows are not sharable"}`))
return
}
// Starting validation of the POST workflow
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Body data error on mail: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
parsedWorkflow := shuffle.Workflow{}
err = json.Unmarshal(body, &parsedWorkflow)
if err != nil {
log.Printf("[WARNING] Unmarshal error on mail: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Super basic validation. Doesn't really matter.
if parsedWorkflow.ID != workflow.ID || len(parsedWorkflow.Actions) != len(workflow.Actions) {
log.Printf("[WARNING] Bad ID during publish: %s vs %s", workflow.ID, parsedWorkflow.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if !workflow.IsValid || !workflow.PreviouslySaved {
log.Printf("[INFO] Failed uploading new workflow because it's invalid or not saved")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid workflows are not sharable"}`))
return
}
workflowData, err := json.Marshal(parsedWorkflow)
if err != nil {
log.Printf("[WARNING] Failed marshalling workflow: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Sanitization is done in the frontend as well
parsedWorkflow = shuffle.SanitizeWorkflow(parsedWorkflow)
parsedWorkflow.ID = uuid.NewV4().String()
action := shuffle.CloudSyncJob{
Type: "workflow",
Action: "publish",
OrgId: user.ActiveOrg.Id,
PrimaryItemId: workflow.ID,
SecondaryItem: string(workflowData),
FifthItem: user.Id,
}
org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed setting getting org during cloud job setting: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
err = executeCloudAction(action, org.SyncConfig.Apikey)
if err != nil {
log.Printf("[WARNING] Failed cloud PUBLISH: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
log.Printf("[INFO] Successfully published workflow %s (%s) TO CLOUD", workflow.Name, workflow.ID)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func handleAppZipUpload(resp http.ResponseWriter, request *http.Request) {
cors := shuffle.HandleCors(resp, request)
if cors {
return
}
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "The upload API is not yet implemented in self-hosted Shuffle. Please explore the app hotloading system: https://shuffler.io/docs/app_creation#uploading-an-app"}`))
return
//https://stackoverflow.com/questions/22964950/http-request-formfile-handle-zip-files
request.ParseMultipartForm(32 << 20)
f, _, err := request.FormFile("shuffle_file")
if err != nil {
log.Printf("[ERROR] Couldn't upload file: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed uploading file. Correct usage is: shuffle_file=@filepath"}`))
return
}
fileSize, err := f.Seek(0, 2) //2 = from end
if err != nil {
panic(err)
}
_, err = f.Seek(0, 0)
if err != nil {
panic(err)
}
buf := new(bytes.Buffer)
fileSize, err = io.Copy(buf, f)
if err != nil {
panic(err)
}
zipdata, err := zip.NewReader(bytes.NewReader(buf.Bytes()), fileSize)
if err != nil {
panic(err)
}
// https://github.com/alexmullins/zip/blob/master/example_test.go
for _, item := range zipdata.File {
log.Printf("\n\nName: %s\n\n", item.FileHeader.Name)
log.Printf("item: %#v", item)
rr, err := item.Open()
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(os.Stdout, rr)
if err != nil {
log.Fatal(err)
}
rr.Close()
}
resp.WriteHeader(200)
resp.Write([]byte("OK"))
}
func initHandlers() {
var err error
ctx := context.Background()
CronScheduler.StartAsync()
log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection")
//requestCache = cache.New(5*time.Minute, 10*time.Minute)
//es := shuffle.GetEsConfig()
elasticConfig := "elasticsearch"
if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) == "false" {
elasticConfig = ""
}
for {
_, err = shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), gceProject, "onprem", true, elasticConfig, false, 0)
if err != nil {
log.Printf("[ERROR] Error in initial database connection. Retrying in 5 seconds. %s", err)
time.Sleep(5 * time.Second)
continue
}
break
}
log.Printf("[DEBUG] Initialized Shuffle database connection. Setting up environment.")
if elasticConfig == "elasticsearch" {
time.Sleep(10 * time.Second)
go runInitEs(ctx)
} else {
//go shuffle.runInit(ctx)
log.Printf("[ERROR] Opensearch is the only viable option. Please set SHUFFLE_ELASTIC=true")
os.Exit(1)
}
r := mux.NewRouter()
r.HandleFunc("/api/v1/_ah/health", shuffle.HealthCheckHandler)
r.HandleFunc("/api/v1/health", shuffle.RunOpsHealthCheck).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/health/stats", shuffle.GetOpsDashboardStats).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/health/opensearch-prefix", shuffle.HandleFixOpensearchPrefix).Methods("POST", "OPTIONS")
// Make user related locations
// Fix user changes with org
r.HandleFunc("/api/v1/users/login", shuffle.HandleLogin).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/register", handleRegister).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/checkusers", checkAdminLogin).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/getinfo", handleInfo).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/{userId}/apps", shuffle.HandleGetUserApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/apps", shuffle.HandleGetUserApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/users/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS")
// r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/{key}/get2fa", shuffle.HandleGet2fa).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/{key}/set2fa", shuffle.HandleSet2fa).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
// General - duplicates and old.
r.HandleFunc("/api/v1/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/login", shuffle.HandleLogin).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/logout", shuffle.HandleLogout).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/me", handleInfo).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/generateapikey", shuffle.HandleApiGeneration).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/getenvironments", shuffle.HandleGetEnvironments).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/setenvironments", shuffle.HandleSetEnvironments).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/docs", shuffle.GetDocList).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/docs/{key}", shuffle.GetDocs).Methods("GET", "OPTIONS")
// Queuebuilder and Workflow streams. First is to update a stream, second to get a stream
// Changed from workflows/streams to streams, as appengine was messing up
// This does not increase the API counter
// Used by frontend
r.HandleFunc("/api/v1/streams", handleSetWorkflowExecution).Methods("POST")
r.HandleFunc("/api/v1/streams/results", handleGetWorkflowExecutionResult).Methods("POST", "OPTIONS")
// Used by orborus
r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET", "POST")
r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST")
// App specific. Partially Singul.
r.HandleFunc("/api/v1/apps/categories", shuffle.GetActiveCategories).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/categories/run", singul.RunCategoryAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{key}/execute", executeSingleAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{key}/run", executeSingleAction).Methods("POST", "OPTIONS")
// Agent / MCP actions
r.HandleFunc("/api/v1/apps/{key}/mcp", runMCPAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/mcp", runMCPAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/agent", runMCPAction).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/apps/categories/run", shuffle.RunCategoryAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/upload", handleAppZipUpload).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/activate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/deactivate", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/distribute", activateWorkflowAppDocker).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.GetFrameworkConfiguration).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/frameworkConfiguration", shuffle.SetFrameworkConfiguration).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}", shuffle.UpdateWorkflowAppConfig).Methods("PATCH", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}", shuffle.DeleteWorkflowApp).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/config", shuffle.GetWorkflowAppConfig).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/run_hotload", handleAppHotloadRequest).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appName}/run_hotload", handleSingleAppHotloadRequest).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/download_remote", LoadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps", getWorkflowApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/apps/search", getSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication", shuffle.GetAppAuthentication).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication", shuffle.AddAppAuthentication).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/authentication/group", shuffle.AddAppAuthenticationGroup).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/authentication/group", shuffle.GetAppAuthenticationGroup).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/authentication/group/{key}", shuffle.DeleteAppAuthenticationGroup).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/authentication/groups", shuffle.AddAppAuthenticationGroup).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/authentication/groups", shuffle.GetAppAuthenticationGroup).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/authentication/groups/{key}", shuffle.DeleteAppAuthenticationGroup).Methods("DELETE", "OPTIONS")
// Related to use-cases that are not directly workflows.
r.HandleFunc("/api/v1/workflows/usecases/{key}", shuffle.HandleGetUsecase).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/usecases", shuffle.LoadUsecases).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/usecases", shuffle.UpdateUsecases).Methods("POST", "OPTIONS")
// Legacy app things
r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/apps", getWorkflowApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS")
// Workflows
// FIXME - implement the queue counter lol
/* Everything below here increases the counters*/
r.HandleFunc("/api/v1/workflows", shuffle.GetWorkflows).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows", shuffle.SetNewWorkflow).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/search", shuffle.HandleWorkflowRunSearch).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/schedules", shuffle.HandleGetSchedules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/executions", shuffle.GetWorkflowExecutions).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/executions/count", shuffle.HandleGetWorkflowRunCount).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/rerun", checkUnfinishedExecution).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", shuffle.AbortExecution).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/download_remote", shuffle.LoadSpecificWorkflows).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/run", executeWorkflow).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflow).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflowUpdate).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/duplicate", shuffle.DuplicateWorkflow).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/recommend", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
// First v2 API
r.HandleFunc("/api/v2/workflows/{key}/executions", shuffle.GetWorkflowExecutionsV2).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v2/workflows/generate/llm", shuffle.HandleWorkflowGenerationResponse).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v2/workflows/edit/llm", shuffle.HandleEditWorkflowWithLLM).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v2/workflows/generate", shuffle.GenerateSingulWorkflows).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v2/datastore", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v2/datastore", shuffle.HandleSetDatastoreKey).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v2/datastore/category/{category_key}", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v2/datastore/automate", shuffle.HandleDatastoreCategoryConfig).Methods("POST", "OPTIONS")
// New for recommendations in Shuffle
r.HandleFunc("/api/v1/recommendations/get_actions", shuffle.HandleActionRecommendation).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/recommendations/modify", shuffle.HandleRecommendationAction).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/revisions", shuffle.GetWorkflowRevisions).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/child_workflows", shuffle.GetChildWorkflows).Methods("GET", "OPTIONS")
// Triggers
r.HandleFunc("/api/v1/hooks/new", shuffle.HandleNewHook).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/hooks", shuffle.HandleNewHook).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS")
r.HandleFunc("/api/v1/hooks/{key}/delete", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/hooks/{key}", shuffle.HandleDeleteHook).Methods("DELETE", "OPTIONS")
// This structure is horrendous. Needs fixing after we got the prototype up
r.HandleFunc("/api/v1/detections", shuffle.HandleListDetectionCategories).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{detectionType}/connect", shuffle.HandleDetectionAutoConnect).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{detection_type}", shuffle.HandleGetDetectionRules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{detection_type}/selected_rules/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/detections/{detection_type}/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS")
// OpenAPI configuration
r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/get_openapi_uri", shuffle.EchoOpenapiData).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/validate_openapi", shuffle.ValidateSwagger).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS")
// Specific triggers
// r.HandleFunc("/api/v1/workflows/{key}/outlook", shuffle.HandleCreateOutlookSub).Methods("POST", "OPTIONS")
// r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", shuffle.HandleDeleteOutlookSub).Methods("DELETE", "OPTIONS")
// r.HandleFunc("/api/v1/triggers/outlook/register", shuffle.HandleNewOutlookRegister).Methods("GET", "OPTIONS")
// r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS")
// r.HandleFunc("/api/v1/triggers/outlook/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS")
// r.HandleFunc("/api/v1/triggers/gmail/register", shuffle.HandleNewGmailRegister).Methods("GET", "OPTIONS")
// r.HandleFunc("/api/v1/triggers/gmail/getFolders", shuffle.HandleGetGmailFolders).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/triggers/pipeline", shuffle.HandleNewPipelineRegister).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/triggers/github/register", shuffle.HandleNewGithubRegister).Methods("PUT", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/pipeline/save", shuffle.HandleSavePipelineInfo).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/pipelines/{key}", handlePipelineCallback).Methods("POST", "GET", "PATCH", "PUT", "DELETE", "OPTIONS")
r.HandleFunc("/api/v1/triggers", shuffle.HandleGetTriggers).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/gmail/routing", handleGmailRouting).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/triggers/gmail/{key}", shuffle.HandleGetSpecificTrigger).Methods("GET", "OPTIONS")
// r.HandleFunc("/api/v1/workflows/{key}/gmail", shuffle.HandleCreateGmailSub).Methods("POST", "OPTIONS")
// r.HandleFunc("/api/v1/workflows/{key}/gmail/{triggerId}", shuffle.HandleDeleteGmailSub).Methods("DELETE", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/gmail/{key}", handleGetSpecificGmailTrigger).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/outlook/getFolders", shuffle.HandleGetOutlookFolders).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/outlook/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/triggers/outlook/{key}/callback", handleOutlookCallback).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS")
// EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021)
r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/create_sub_org", shuffle.HandleCreateSubOrg).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/change", shuffle.HandleChangeUserOrg).Methods("POST", "OPTIONS") // Swaps to the org
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleDeleteOrg).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/suborgs", shuffle.HandleGetSubOrgs).Methods("GET", "OPTIONS")
// This is a new API that validates if a key has been seen before.
// Not sure what the best course of action is for it.
r.HandleFunc("/api/v1/getenvironments", shuffle.HandleGetEnvironments).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/setenvironments", shuffle.HandleSetEnvironments).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/environments/{key}/stop", shuffle.HandleStopExecutions).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/environments/{key}/rerun", shuffle.HandleRerunExecutions).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/environments/{key}/stats", shuffle.HandleGetenvStats).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/environments/{key}/config", shuffle.HandleSetenvConfig).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/environments", shuffle.HandleGetEnvironments).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/validate_app_values", shuffle.HandleKeyValueCheck).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/list_cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleGetCacheKey).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/get_cache", shuffle.HandleGetCacheKey).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/set_cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/delete_cache", shuffle.HandleDeleteCacheKeyPost).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache/config", shuffle.HandleCacheConfig).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/statistics", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/stats", shuffle.HandleGetStatistics).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/stats", shuffle.HandleAppendStatistics).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/stats/{key}", shuffle.GetSpecificStats).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/cache/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/datastore", shuffle.HandleListCacheKeys).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/datastore", shuffle.HandleSetCacheKey).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}/datastore/{cache_key}", shuffle.HandleDeleteCacheKey).Methods("DELETE", "OPTIONS")
// Docker orborus specific - downloads an image
r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "GET", "OPTIONS")
r.HandleFunc("/api/v1/login_sso", shuffle.HandleSAML).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "POST", "OPTIONS")
// Important for email, IDS etc. Create this by:
// PS: For cloud, this has to use cloud storage.
// https://developer.box.com/reference/get-files-id-content/
r.HandleFunc("/api/v1/files/download_remote", shuffle.HandleDownloadRemoteFiles).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/namespaces/{namespace}", shuffle.HandleGetFileNamespace).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS", "PATCH")
r.HandleFunc("/api/v1/files/{fileId}/edit", shuffle.HandleEditFile).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/files/{fileId}/config", shuffle.HandleSetFileConfig).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/files/namespaces/{namespace}/share", shuffle.HandleShareNamespace).Methods("POST", "OPTIONS")
// This structure is horrendous. Needs fixing after we got the prototype up
r.HandleFunc("/api/v1/detections/{detectionType}/connect", shuffle.HandleDetectionAutoConnect).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{detection_type}", shuffle.HandleGetDetectionRules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules", shuffle.HandleGetSelectedRules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/detections/{triggerId}/selected_rules/save", shuffle.HandleSaveSelectedRules).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/detections/{action}", shuffle.HandleFolderToggle).Methods("PUT", "OPTIONS")
// This is weird.
r.HandleFunc("/api/v1/detections/{fileId}/{action}", shuffle.HandleToggleRule).Methods("PUT", "OPTIONS")
//r.HandleFunc("/api/v1/detections/siem/node_health", shuffle.HandleTenzirHealthUpdate).Methods("POST","OPTIONS")
// Introduced in 0.9.21 to handle notifications for e.g. failed Workflow
r.HandleFunc("/api/v1/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/notifications", shuffle.HandleCreateNotification).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/conversation", shuffle.RunActionAI).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/dashboards/{key}/widgets", shuffle.HandleNewWidget).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/dashboards/{key}/widgets/{widget_id}", shuffle.HandleGetWidget).Methods("GET", "OPTIONS")
// Need to add auth in pprof
// if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" || strings.ToLower(os.Getenv("DEBUG_MEMORY")) == "true" {
// log.Printf("[DEBUG] Memory debugging is enabled on /debug/pprof")
// r.HandleFunc("/debug/pprof/", pprof.Index)
// r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
// r.HandleFunc("/debug/pprof/profile", pprof.Profile)
// r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
// r.HandleFunc("/debug/pprof/trace", pprof.Trace)
// } else {
// log.Printf("[DEBUG] Memory debugging is disabled. To enable, set SHUFFLE_DEBUG_MEMORY or DEBUG_MEMORY to true")
// }
r.Use(shuffle.RequestMiddleware)
http.Handle("/", r)
}
// Had to move away from mux, which means Method is fucked up right now.
func main() {
if os.Getenv("DEBUG") == "true" {
debug = true
}
initHandlers()
hostname, err := os.Hostname()
if err != nil {
hostname = "MISSING"
}
innerPort := os.Getenv("BACKEND_PORT")
if innerPort == "" {
log.Printf("[DEBUG] Running on %s:5001", hostname)
log.Fatal(http.ListenAndServe(":5001", nil))
os.Setenv("BACKEND_PORT", "5001")
} else {
log.Printf("[DEBUG] Running on %s:%s", hostname, innerPort)
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", innerPort), nil))
}
}
Reference in New Issue View Git Blame Copy Permalink