package shuffle
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
"crypto/sha256"
"sync"
"hash/fnv"
neturl "net/url"
"path"
"sort"
"unicode"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"google.golang.org/api/cloudfunctions/v1"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
scheduler "cloud.google.com/go/scheduler/apiv1"
"cloud.google.com/go/scheduler/apiv1/schedulerpb"
"cloud.google.com/go/storage"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
http2 "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
"gopkg.in/yaml.v3"
"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/shirou/gopsutil/v3/process"
"regexp"
"strconv"
"strings"
"time"
"encoding/base32"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
// "github.com/goccy/go-json"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/md5"
"crypto/rand"
"crypto/sha1"
mathrand "math/rand"
"github.com/bradfitz/slice"
uuid "github.com/satori/go.uuid"
"github.com/sendgrid/sendgrid-go"
qrcode "github.com/skip2/go-qrcode"
"github.com/frikky/kin-openapi/openapi2"
"github.com/frikky/kin-openapi/openapi2conv"
"github.com/frikky/kin-openapi/openapi3"
"github.com/google/go-github/v28/github"
"golang.org/x/crypto/bcrypt"
"golang.org/x/oauth2"
"github.com/Masterminds/semver"
"runtime"
dockerclient "github.com/docker/docker/client"
)
var project ShuffleStorage
var baseDockerName = "frikky/shuffle"
var SSOUrl = ""
var kmsDebug = false
var debug = os.Getenv("DEBUG") == "true"
var sandboxProject = "shuffle-sandbox-337810"
func GetProject() ShuffleStorage {
return project
}
// Injects the header in all requests
func RequestMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
next.ServeHTTP(w, r)
})
}
// In case we need custom context control in the future
// This is used ~everywhere, and used to exist due to GCP AppEngine's custom
// context handling
func GetContext(request *http.Request) context.Context {
return context.Background()
}
func HandleCors(resp http.ResponseWriter, request *http.Request) bool {
origin := request.Header["Origin"]
resp.Header().Set("Vary", "Origin")
if project.Environment == "cloud" {
allowedDomains := []string{
"https://shuffler.io",
"https://stream.shuffler.io",
"https://us.shuffler.io",
"https://california.shuffler.io",
"https://eu.shuffler.io",
"https://frankfurt.shuffler.io",
"https://ca.shuffler.io",
"https://canada.shuffler.io",
"https://au.shuffler.io",
"https://jp.shuffler.io",
"https://br.shuffler.io",
"https://in.shuffler.io",
// Related projects (maybe)
"https://*.singul.io",
"https://singul.io",
"https://*.shutdown.no",
"https://shutdown.no",
// Local testing
"http://localhost:3002",
"http://localhost:3000",
// Shuffle support
"https://cases.shuffler.io",
"https://security.shuffler.io",
"https://id-preview--83c56bc8-506d-4dc5-a245-6b57e03ff019.lovable.app",
// tbd
"https://preview--shuffle-cases.lovable.app",
"https://9f29a11a-6489-4898-8044-ed7b8f848ef9.lovableproject.com",
"https://id-preview--9f29a11a-6489-4898-8044-ed7b8f848ef9.lovable.app",
// Support project
"https://support.shuffler.io",
"https://compliance.shuffler.io",
"https://2538a36b-5c1c-4954-8700-ee5d6c6b9f91.lovableproject.com",
"https://shuffle-support.lovable.app",
"https://shuffle-support.lovable.app/",
"https://05364669-00ea-43be-ae8f-8e333ccc870c.lovableproject.com",
"https://preview--shuffle-support.lovable.app",
}
if len(origin) > 0 {
// Check if the origin is in the allowed domains
allowed := false
for _, domain := range allowedDomains {
if origin[0] == domain {
allowed = true
break
}
}
// Since we are becoming more and more of a platform
if !allowed {
currentUrl := strings.ToLower(request.URL.String())
allowedUrls := []string{"/api/v1/", "/api/v2/"}
disallowedUrls := []string{"/settings", "/register", "/login_openid", "/login_sso"}
for _, allowedUrl := range allowedUrls {
if !strings.HasPrefix(currentUrl, allowedUrl) {
continue
}
allowed = true
for _, disallowedUrl := range disallowedUrls {
if strings.HasSuffix(currentUrl, disallowedUrl) {
allowed = false
break
}
}
break
}
}
if allowed {
resp.Header().Set("Access-Control-Allow-Origin", origin[0])
}
}
} else {
if len(origin) > 0 {
resp.Header().Set("Access-Control-Allow-Origin", origin[0])
} else {
resp.Header().Set("Access-Control-Allow-Origin", "http://localhost:4201")
}
}
//resp.Header().Set("Access-Control-Allow-Origin", "http://localhost:8000")
resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me, Org-Id, Org, Authorization, X-Debug-Url")
resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE, PATCH")
resp.Header().Set("Access-Control-Allow-Credentials", "true")
if request.Method == "OPTIONS" {
resp.WriteHeader(200)
resp.Write([]byte("OK"))
return true
}
return false
}
func Md5sum(data []byte) string {
hasher := md5.New()
hasher.Write(data)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
return newmd5
}
func Md5sumfile(filepath string) string {
dat, err := ioutil.ReadFile(filepath)
if err != nil {
log.Printf("Error in dat: %s", err)
}
hasher := md5.New()
hasher.Write(dat)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
log.Printf("%s: %s", filepath, newmd5)
return newmd5
}
func randStr(strSize int, randType string) string {
var dictionary string
if randType == "alphanum" {
dictionary = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
}
if randType == "alpha" {
dictionary = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
}
if randType == "number" {
dictionary = "0123456789"
}
var bytes = make([]byte, strSize)
rand.Read(bytes)
for k, v := range bytes {
bytes[k] = dictionary[v%byte(len(dictionary))]
}
return string(bytes)
}
func isLoop(arg string) bool {
if strings.Contains(arg, "$") && (strings.HasSuffix(arg, ".#") || strings.Contains(arg, ".#.")) {
return true
}
if strings.Contains(arg, "$") && strings.Contains(arg, ".#") {
pattern := `(^|\.)(#(\d+-\d+)?($|\.))`
re := regexp.MustCompile(pattern)
return strings.Contains(arg, "$") && re.MatchString(arg)
}
return false
}
func ConstructSessionCookie(value string, expires time.Time) *http.Cookie {
c := http.Cookie{
Name: "session_token",
Value: value,
Expires: expires,
Path: "/",
HttpOnly: true,
Secure: false,
Domain: "",
}
if os.Getenv("SHUFFLE_COOKIE_SECURE") == "true" {
c.Secure = true
}
d := os.Getenv("SHUFFLE_COOKIE_DOMAIN")
if len(d) > 0 {
c.Domain = d
}
if project.Environment == "cloud" {
c.Domain = ".shuffler.io"
c.Secure = true
//c.SameSite = http.SameSiteLaxMode
c.SameSite = http.SameSiteNoneMode
}
return &c
}
func constructSessionDeleteCookie() *http.Cookie {
c := ConstructSessionCookie("", time.Time{})
c.MaxAge = -1
return c
}
func HandleSet2fa(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
ctx := GetContext(request)
var user User
var userId string
userSettingUpMfa := false
user, err := HandleApiAuthentication(resp, request)
if err != nil {
parts := strings.Split(request.URL.Path, "/")
if len(parts) < 5 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid URL path."}`))
return
}
MFACode := parts[4]
// Retrieve user ID and unique code from cache
cacheUserId, err := GetCache(ctx, fmt.Sprintf("user_id_%s", MFACode))
if err != nil {
log.Printf("[ERROR] Failed to retrieve user ID from cache: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to retrieve user ID from cache."}`))
return
}
cacheUniqueCode, err := GetCache(ctx, fmt.Sprintf("mfa_code_%s", MFACode))
if err != nil {
log.Printf("[ERROR] Failed to retrieve mfa code from cache: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to retrieve MFA code from cache."}`))
return
}
//if user id and unique code are not empty, user is setting up MFA
if len(cacheUserId.([]byte)) > 0 && len(cacheUniqueCode.([]byte)) > 0 {
userSettingUpMfa = true
}
if mfaCodeBytes, ok := cacheUniqueCode.([]byte); ok {
cacheUniqueCode = string(mfaCodeBytes)
}
//Both unique code present in cache and MFA code token present in url request must match
if cacheUniqueCode != MFACode {
log.Printf("[ERROR] user_id or uniqueId does not match")
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "user_id or uniqueId does not match."}`))
return
}
if userIdBytes, ok := cacheUserId.([]byte); ok {
userId = string(userIdBytes)
}
}
var cacheUser *User
// check if user id received from cache is not empty
if len(userId) > 0 && userSettingUpMfa == true {
cacheUser, err = GetUser(ctx, userId)
if err != nil {
log.Printf("[ERROR] Failed to retrieve user from cache: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to retrieve user from cache."}`))
return
}
}
//if user id is empty, use the user data from cache
if len(user.Id) == 0 {
user = *cacheUser
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting SET 2fa request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
DeleteCache(ctx, fmt.Sprintf("Organizations_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", strings.ToLower(user.Username)))
DeleteCache(ctx, fmt.Sprintf("user_%s", strings.ToLower(user.Id)))
return
}
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 && userSettingUpMfa == false {
log.Printf("[ERROR] Path too short (2fa): %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type parsedValue struct {
Code string `json:"code"`
UserId string `json:"user_id"`
}
var tmpBody parsedValue
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("[WARNING] Error with unmarshal tmpBody in verify 2fa: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(tmpBody.Code) != 6 {
log.Printf("[WARNING] Length of code isn't 6: %s", tmpBody.Code)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Length of code must be 6"}`)))
return
}
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "Failed getting your org."}`))
return
}
// FIXME: Everything should match?
// || user.Id != tmpBody.UserId
if user.Id != fileId && userSettingUpMfa == false {
log.Printf("[WARNING] Bad ID: %s vs %s", user.Id, fileId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can only set 2fa for your own user. Pass field user_id in JSON."}`)))
return
}
foundUser, err := GetUser(ctx, user.Id)
if err != nil {
log.Printf("[ERROR] Can't find user %s (set 2fa): %s", user.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting your user."}`)))
return
}
//https://www.gojek.io/blog/a-diy-two-factor-authenticator-in-golang
interval := time.Now().Unix() / 30
HOTP, err := getHOTPToken(foundUser.MFA.PreviousCode, interval)
if err != nil {
log.Printf("[ERROR] Failed generating a HOTP token: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
if HOTP != tmpBody.Code {
log.Printf("[DEBUG] Bad code sent for user %s (%s). Sent: %s, Want: %s", user.Username, user.Id, tmpBody.Code, HOTP)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Wrong code. Try again"}`))
return
}
MFAActive := false
if foundUser.MFA.Active == true {
foundUser.MFA.Active = false
foundUser.MFA.PreviousCode = foundUser.MFA.ActiveCode
foundUser.MFA.ActiveCode = ""
MFAActive = false
log.Printf("[DEBUG] Successfully disable 2FA authentication for user %s (%s)", foundUser.Username, foundUser.Id)
} else {
foundUser.MFA.Active = true
foundUser.MFA.ActiveCode = foundUser.MFA.PreviousCode
foundUser.MFA.PreviousCode = ""
MFAActive = true
log.Printf("[DEBUG] Successfully Enable 2FA authentication for user %s (%s)", foundUser.Username, foundUser.Id)
}
err = SetUser(ctx, foundUser, true)
if err != nil {
log.Printf("[WARNING] Failed SETTING MFA for user %s (%s): %s", foundUser.Username, foundUser.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed updating your user. Please try again."}`))
return
}
// log.Printf("[DEBUG] Successfully enabled 2FA for user %s (%s)", foundUser.Username, foundUser.Id)
// If user is setting up MFA, than reset the user session or create a new one
if userSettingUpMfa {
user.LoginInfo = append(user.LoginInfo, LoginInfo{
IP: GetRequestIp(request),
Timestamp: time.Now().Unix(),
})
tutorialsFinished := []Tutorial{}
for _, tutorial := range user.PersonalInfo.Tutorials {
tutorialsFinished = append(tutorialsFinished, Tutorial{
Name: tutorial,
})
}
if len(org.SecurityFramework.SIEM.Name) > 0 || len(org.SecurityFramework.Network.Name) > 0 || len(org.SecurityFramework.EDR.Name) > 0 || len(org.SecurityFramework.Cases.Name) > 0 || len(org.SecurityFramework.IAM.Name) > 0 || len(org.SecurityFramework.Assets.Name) > 0 || len(org.SecurityFramework.Intel.Name) > 0 || len(org.SecurityFramework.Communication.Name) > 0 {
tutorialsFinished = append(tutorialsFinished, Tutorial{
Name: "find_integrations",
})
}
for _, tutorial := range org.Tutorials {
tutorialsFinished = append(tutorialsFinished, tutorial)
}
//log.Printf("[INFO] Tutorials finished: %v", tutorialsFinished)
returnValue := HandleInfo{
Success: true,
Tutorials: tutorialsFinished,
}
loginData := `{"success": true}`
newData, err := json.Marshal(returnValue)
if err == nil {
loginData = string(newData)
}
if len(user.Session) != 0 {
log.Printf("[INFO] User session exists - resetting session")
expiration := time.Now().Add(8 * time.Hour)
newCookie := ConstructSessionCookie(user.Session, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
//log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", user.Session)
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "session_token",
Value: user.Session,
Expiration: expiration.Unix(),
})
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "__session",
Value: user.Session,
Expiration: expiration.Unix(),
})
loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, user.Session, expiration.Unix())
newData, err := json.Marshal(returnValue)
if err == nil {
loginData = string(newData)
}
err = SetSession(ctx, user, user.Session)
if err != nil {
log.Printf("[WARNING] Error adding session to database: %s", err)
} else {
//log.Printf("[DEBUG] Updated session in backend")
}
user.MFA = foundUser.MFA
err = SetUser(ctx, &user, false)
if err != nil {
log.Printf("[ERROR] Failed updating user when setting session (2): %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(loginData))
return
} else {
log.Printf("[INFO] User session for %s (%s) is empty - create one!", user.Username, user.Id)
sessionToken := uuid.NewV4().String()
expiration := time.Now().Add(8 * time.Hour)
newCookie := ConstructSessionCookie(sessionToken, expiration)
// Does it not set both?
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
// ADD TO DATABASE
err = SetSession(ctx, user, sessionToken)
if err != nil {
log.Printf("[DEBUG] Error adding session to database: %s", err)
}
user.Session = sessionToken
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "session_token",
Value: sessionToken,
Expiration: expiration.Unix(),
})
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "__session",
Value: sessionToken,
Expiration: expiration.Unix(),
})
user.MFA = foundUser.MFA
err = SetUser(ctx, &user, true)
if err != nil {
log.Printf("[ERROR] Failed updating user when setting session: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, sessionToken, expiration.Unix())
newData, err := json.Marshal(returnValue)
if err == nil {
loginData = string(newData)
}
}
log.Printf("[INFO] %s SUCCESSFULLY LOGGED IN with session %s", user.Username, user.Session)
resp.WriteHeader(200)
resp.Write([]byte(loginData))
return
}
response := fmt.Sprintf(`{"success": true, "reason": "Correct code. MFA is now required for this user.", "MFAActive": %v}`, MFAActive)
resp.WriteHeader(200)
resp.Write([]byte(response))
}
func getHOTPToken(secret string, interval int64) (string, error) {
//Converts secret to base32 Encoding. Base32 encoding desires a 32-character
//subset of the twenty-six letters A–Z and ten digits 0–9
key, err := base32.StdEncoding.DecodeString(strings.ToUpper(secret))
if err != nil {
return "", err
}
bs := make([]byte, 8)
binary.BigEndian.PutUint64(bs, uint64(interval))
//Signing the value using HMAC-SHA1 Algorithm
hash := hmac.New(sha1.New, key)
hash.Write(bs)
h := hash.Sum(nil)
// We're going to use a subset of the generated hash.
// Using the last nibble (half-byte) to choose the index to start from.
// This number is always appropriate as it's maximum decimal 15, the hash will
// have the maximum index 19 (20 bytes of SHA1) and we need 4 bytes.
o := (h[19] & 15)
var header uint32
//Get 32 bit chunk from hash starting at the o
r := bytes.NewReader(h[o : o+4])
err = binary.Read(r, binary.BigEndian, &header)
if err != nil {
return "", err
}
//Ignore most significant bits as per RFC 4226.
//Takes division from one million to generate a remainder less than < 7 digits
h12 := (int(header) & 0x7fffffff) % 1000000
//Converts number as a string
otp := strconv.Itoa(int(h12))
// Dumb solutions <3
// This works well, as the numbers are small ^_^
if len(otp) == 0 {
otp = "000000"
} else if len(otp) == 1 {
otp = fmt.Sprintf("00000%s", otp)
} else if len(otp) == 2 {
otp = fmt.Sprintf("0000%s", otp)
} else if len(otp) == 3 {
otp = fmt.Sprintf("000%s", otp)
} else if len(otp) == 4 {
otp = fmt.Sprintf("00%s", otp)
} else if len(otp) == 5 {
otp = fmt.Sprintf("0%s", otp)
}
return otp, nil
}
func HandleGet2fa(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting GET 2fa request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
ctx := GetContext(request)
var user User
var userId string
userSettingUpMfa := false
user, err := HandleApiAuthentication(resp, request)
if err != nil {
// Attempt to retrieve user data from cache
parts := strings.Split(request.URL.Path, "/")
if len(parts) < 5 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid URL path."}`))
return
}
MFACode := parts[4]
// Retrieve user ID and unique code from cache
cacheUserId, err := GetCache(ctx, fmt.Sprintf("user_id_%s", MFACode))
if err != nil {
log.Printf("[ERROR] Failed to retrieve user ID from cache: %s", err)
resp.WriteHeader(404)
resp.Write([]byte(`{"success": false, "reason": "Failed to retrieve user ID from cache."}`))
return
}
cacheUniqueCode, err := GetCache(ctx, fmt.Sprintf("mfa_code_%s", MFACode))
if err != nil {
log.Printf("[ERROR] Failed to retrieve mfa code from cache: %s", err)
resp.WriteHeader(404)
resp.Write([]byte(`{"success": false, "reason": "Failed to retrieve MFA code from cache."}`))
return
}
//if user id and unique code are not empty, user is setting up MFA
if len(cacheUserId.([]byte)) > 0 && len(cacheUniqueCode.([]byte)) > 0 {
userSettingUpMfa = true
}
if mfaCodeBytes, ok := cacheUniqueCode.([]byte); ok {
cacheUniqueCode = string(mfaCodeBytes)
}
if userIdBytes, ok := cacheUserId.([]byte); ok {
userId = string(userIdBytes)
}
//Both unique code present in cache and MFA code token present in url request must match
if cacheUniqueCode != MFACode {
log.Printf("[ERROR] Invalid user for the MFA code %s", MFACode)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "Invalid user for the MFA code."}`))
return
}
}
var cacheUser *User
// check if user id received from cache is not empty
if len(userId) > 0 && userSettingUpMfa == true {
cacheUser, err = GetUser(ctx, userId)
if err != nil {
log.Printf("[ERROR] Failed to retrieve user from cache: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to retrieve user from cache."}`))
return
}
}
//if user id is empty, use the user data from cache
if len(user.Id) == 0 {
user = *cacheUser
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" && userSettingUpMfa == false {
if len(location) <= 4 {
log.Printf("[ERROR] Path too short - MFA: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if user.Id != fileId && userSettingUpMfa == false {
log.Printf("[WARNING] Bad ID: %s vs %s", user.Id, fileId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can only set 2fa for your own user"}`)))
return
}
// https://socketloop.com/tutorials/golang-generate-qr-codes-for-google-authenticator-app-and-fix-cannot-interpret-qr-code-error
// generate a random string - preferably 6 or 8 characters
randomStr := randStr(8, "alphanum")
// For Google Authenticator purpose
// for more details see
// https://github.com/google/google-authenticator/wiki/Key-Uri-Format
secret := base32.StdEncoding.EncodeToString([]byte(randomStr))
// authentication link. Remember to replace SocketLoop with yours.
// for more details see
// https://github.com/google/google-authenticator/wiki/Key-Uri-Format
authLink := fmt.Sprintf("otpauth://totp/%s?secret=%s&issuer=Shuffle", user.Username, secret)
png, err := qrcode.Encode(authLink, qrcode.Medium, 256)
if err != nil {
log.Printf("[ERROR] Failed PNG encoding: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed image encoding"}`)))
return
}
dataURI := fmt.Sprintf("data:image/png;base64,%s", base64.StdEncoding.EncodeToString([]byte(png)))
newres := ResultChecker{
Success: true,
Reason: dataURI,
Extra: strings.ReplaceAll(secret, "=", "A"),
}
newjson, err := json.Marshal(newres)
if err != nil {
log.Printf("[ERROR] Failed marshal in get OTP: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking data"}`)))
return
}
//user.MFA.PreviousCode = authLink
user.MFA.PreviousCode = secret
err = SetUser(ctx, &user, true)
if err != nil {
log.Printf("[WARNING] Failed updating MFA for user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed updating your user"}`))
return
}
log.Printf("[DEBUG] Sent new MFA update for user %s (%s)", user.Username, user.Id)
//log.Printf("%s", newjson)
resp.WriteHeader(200)
resp.Write([]byte(newjson))
}
func HandleGetOrgs(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting GET ORGS request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get orgs: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
if user.Role != "global_admin" {
orgs := []OrgMini{}
for _, item := range user.Orgs {
// FIXM: Should return normal orgs, but hidden if the user isn't admin
org, err := GetOrg(ctx, item)
if err == nil {
orgs = append(orgs, OrgMini{
Id: org.Id,
Name: org.Name,
CreatorOrg: org.CreatorOrg,
Image: org.Image,
})
// Role: "admin",
}
}
newjson, err := json.Marshal(orgs)
if err != nil {
log.Printf("[WARNING] Failed marshal in get orgs: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`)))
return
}
//log.Printf("[AUDIT] User %s (%s) isn't global admin and can't list orgs. Returning list of local orgs.", user.Username, user.Id)
resp.WriteHeader(200)
resp.Write([]byte(newjson))
return
}
orgs, err := GetAllOrgs(ctx)
if err != nil || len(orgs) == 0 {
log.Printf("[WARNING] Failed getting orgs: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't get orgs"}`))
return
}
newjson, err := json.Marshal(orgs)
if err != nil {
log.Printf("[WARNING] Failed unmarshal in get orgs: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func HandleGetOrg(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Checking if it's a special region. All user-specific requests should
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting GET ORG request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short (getorg): %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
ctx := GetContext(request)
sanitizeOrg := false
user, err := HandleApiAuthentication(resp, request)
if err != nil {
// This is specifically for public workflows
referenceId, referenceok := request.URL.Query()["reference_execution"]
authorization, authorizationok := request.URL.Query()["authorization"]
if referenceok && authorizationok {
workflowExecution, err := GetWorkflowExecution(ctx, referenceId[0])
if err == nil && authorization[0] == workflowExecution.Authorization {
sanitizeOrg = true
}
}
if sanitizeOrg != true {
log.Printf("[WARNING] Api authentication failed in get org: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
org, err := GetOrg(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", fileId, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org details"}`))
return
}
// clean getOrg invites
org.Invites = []string{}
if org.OrgAuth.Token == "" {
org.OrgAuth.Token = uuid.NewV4().String()
org.OrgAuth.Expires = time.Now().AddDate(0, 0, 1)
SetOrg(ctx, *org, org.Id)
}
// Check if orgauth is expired
if org.OrgAuth.Expires.Before(time.Now()) {
if debug {
log.Printf("[DEBUG] Refreshing org token for %s (%s)", org.Name, org.Id)
}
org.OrgAuth.Token = uuid.NewV4().String()
org.OrgAuth.Expires = time.Now().AddDate(0, 0, 1)
SetOrg(ctx, *org, org.Id)
}
admin := false
if user.SupportAccess == true {
admin = true
sanitizeOrg = false
// Update active org for user to this one?
// This makes it possible to walk around in the UI for the org
/*
if user.ActiveOrg.Id != org.Id {
log.Printf("[AUDIT] User %s (%s) is admin and has access to org %s. Updating active org to this one.", user.Username, user.Id, org.Id)
user.ActiveOrg.Id = org.Id
user.ActiveOrg.Name = org.Name
user.Role = "admin"
SetUser(ctx, &user, false)
DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
}
*/
} else {
userFound := false
for _, inneruser := range org.Users {
if inneruser.Id == user.Id {
userFound = true
if inneruser.Role == "admin" {
admin = true
}
break
}
}
if !userFound && !sanitizeOrg {
found := false
for _, orgId := range user.Orgs {
if orgId == org.Id {
found = true
admin = false
break
}
}
if !found {
log.Printf("[ERROR] User '%s' (%s) isn't a part of org %s (%s) (get org)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User doesn't have access to org"}`))
return
}
}
}
if !admin {
org.Defaults = Defaults{}
// Clean sensitive SSO fields instead of clearing entire config
if org.SSOConfig.OpenIdClientId != "" {
org.SSOConfig.OpenIdClientId = "CLEANED"
}
if org.SSOConfig.OpenIdClientSecret != "" {
org.SSOConfig.OpenIdClientSecret = "CLEANED"
}
org.Subscriptions = []PaymentSubscription{}
org.ManagerOrgs = []OrgMini{}
org.ChildOrgs = []OrgMini{}
org.Invites = []string{}
org.OrgAuth = OrgAuth{}
org.Billing = Billing{}
} else {
org.SyncFeatures.AppExecutions.Description = "The amount of Apps within Workflows you can run per month. This limit can be exceeded when running workflows without a trigger (manual execution). Usage resets monthly."
org.SyncFeatures.WorkflowExecutions.Description = "N/A. See App Executions"
org.SyncFeatures.Webhook.Description = "Webhooks are Triggers that take an HTTP input to start a workflow. Read docs for more."
org.SyncFeatures.Schedules.Description = "Schedules are Triggers that run on an interval defined by you. Read docs for more."
org.SyncFeatures.MultiEnv.Description = "Multiple Environments are used to run automation in different physical locations. Change from /admin?tab=environments"
org.SyncFeatures.MultiTenant.Description = "Multiple Tenants can be used to segregate information for each MSSP Customer. Change from /admin?tab=suborgs"
org.SyncFeatures.MultiRegion.Description = "Multiregion allows you to change region to our other data centers around the world."
org.SyncFeatures.SendSms.Description = "Allows you to send SMS through Shuffle Tools or our API. Usage resets monthly."
org.SyncFeatures.SendMail.Description = "Allows you to send email through Shuffle Tools or our API. Usage resets monthly."
//org.SyncFeatures.MultiTenant.Description = "Multiple Tenants can be used to segregate information for each MSSP Customer. Change from /admin?tab=suborgs"
//log.Printf("LIMIT: %s", org.SyncFeatures.AppExecutions.Limit)
orgChanged := false
if org.SyncFeatures.AppExecutions.Limit == 0 || org.SyncFeatures.AppExecutions.Limit == 1500 {
org.SyncFeatures.AppExecutions.Limit = 2000
orgChanged = true
}
if org.SyncFeatures.SendMail.Limit == 0 {
org.SyncFeatures.SendMail.Limit = 100
orgChanged = true
}
if org.SyncFeatures.SendSms.Limit == 0 {
org.SyncFeatures.SendSms.Limit = 30
orgChanged = true
}
org.SyncFeatures.EmailTrigger.Limit = 0
if org.SyncFeatures.MultiEnv.Limit == 0 {
org.SyncFeatures.MultiEnv.Limit = 1
orgChanged = true
}
org.SyncFeatures.EmailTrigger.Limit = 0
org.SyncFeatures.MultiTenant.Usage = int64(len(org.ChildOrgs) + 1)
if org.SyncUsage.MultiTenant.Counter != int64(len(org.ChildOrgs)+1) {
org.SyncUsage.MultiTenant.Counter = int64(len(org.ChildOrgs) + 1)
orgChanged = true
}
if len(org.CreatorOrg) == 0 {
allChildOrgs, _, err := GetAllChildOrgs(ctx, org.Id)
if err == nil {
if len(allChildOrgs) != len(org.ChildOrgs) {
allChildOrgsMini := []OrgMini{}
for _, child := range allChildOrgs {
allChildOrgsMini = append(allChildOrgsMini, OrgMini{
Id: child.Id,
Name: child.Name,
CreatorOrg: child.CreatorOrg,
Image: "",
RegionUrl: child.RegionUrl,
})
}
org.ChildOrgs = allChildOrgsMini
org.SyncFeatures.MultiTenant.Usage = int64(len(allChildOrgsMini) + 1)
org.SyncUsage.MultiTenant.Counter = int64(len(allChildOrgsMini) + 1)
orgChanged = true
}
}
}
if orgChanged {
log.Printf("[DEBUG] Org features for %s (%s) changed. Updating.", org.Name, org.Id)
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org during org loading")
}
}
info, err := GetOrgStatistics(ctx, fileId)
if err == nil {
org.SyncFeatures.AppExecutions.Usage = info.MonthlyAppExecutions
}
envs, err := GetEnvironments(ctx, fileId)
if err == nil {
//log.Printf("Envs: %s", len(envs))
org.SyncFeatures.MultiEnv.Usage = int64(len(envs))
}
// Backfill subscription IDs if any subscription is missing an ID
addSubId := false
for _, sub := range org.Subscriptions {
if sub.Id == "" {
addSubId = true
break
}
}
if addSubId {
for i := range org.Subscriptions {
if org.Subscriptions[i].Id == "" {
org.Subscriptions[i].Id = uuid.NewV4().String()
}
}
if err := SetOrg(ctx, *org, org.Id); err != nil {
log.Printf("[WARNING] Failed to backfill subscription IDs for org %s: %s", org.Id, err)
} else {
log.Printf("[INFO] Backfilled subscription IDs for org %s", org.Id)
}
}
if len(org.Subscriptions) == 0 && len(org.CreatorOrg) == 0 {
// Only when there is no subscription in the org and it's not a suborg :)
// Placeholder subscription that to add at very first time
base := BuildBaseSubscription(*org, org.SyncFeatures.AppExecutions.Limit)
org.Subscriptions = append(org.Subscriptions, base)
if err := SetOrg(ctx, *org, org.Id); err != nil {
log.Printf("[WARNING] Failed to persist base subscription for org %s: %s", org.Id, err)
} else {
log.Printf("[INFO] Added a base subscription (%s) for org %s", base.Name, org.Id)
}
} else if len(org.CreatorOrg) == 0 && len(org.Subscriptions) >= 1 {
hasActivePaidSubscription := false
hasFreeSubscription := false
updateSub := false
for _, sub := range org.Subscriptions {
if sub.Active && sub.Amount != "0" {
hasActivePaidSubscription = true
}
if sub.Amount == "0" && sub.Reference == "" {
hasFreeSubscription = true
}
}
if hasActivePaidSubscription && hasFreeSubscription {
// Remove free subscriptions since user has active paid plan
var filteredSubs []PaymentSubscription
for _, sub := range org.Subscriptions {
if !(sub.Amount == "0" && sub.Reference == "") {
filteredSubs = append(filteredSubs, sub)
}
}
org.Subscriptions = filteredSubs
updateSub = true
log.Printf("[INFO] Removed free subscription for org %s (active paid subscription exists)", org.Id)
} else if !hasActivePaidSubscription && !hasFreeSubscription {
// No active paid subscription and no free plan, add one
org.Subscriptions = append(org.Subscriptions, BuildBaseSubscription(*org, 2000))
updateSub = true
log.Printf("[INFO] Added free subscription for org %s (no active paid subscriptions found)", org.Id)
}
// Persist any subscription changes made above
if updateSub {
if err := SetOrg(ctx, *org, org.Id); err != nil {
log.Printf("[ERROR] Failed to persist subscription changes for org %s: %v", org.Id, err)
} else {
log.Printf("[DEBUG] Successfully persisted subscription changes for org %s", org.Id)
}
}
}
if len(org.CreatorOrg) == 0 && project.Environment == "onprem" {
// This is used to update the subscription for the onprem orgs
// That have cloud sync active
// Not a suborg
cloudOrg := HandleCheckLicense(ctx, *org)
org = &cloudOrg
} else if len(org.CreatorOrg) > 0 && project.Environment == "onprem" {
parentOrg, err := GetOrg(ctx, org.CreatorOrg)
if err == nil && len(parentOrg.Subscriptions) > 0 {
licenseOrg := HandleCheckLicense(ctx, *parentOrg)
parentOrg = &licenseOrg
org.Subscriptions = parentOrg.Subscriptions
}
}
if len(org.CreatorOrg) == 0 && project.Environment == "onprem" {
parentOrg := HandleCheckLicense(ctx, *org)
org = &parentOrg
}
}
if project.Environment == "onprem" {
statistics, err := GetOrgStatistics(ctx, org.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org statistics for %s: %s", org.Id, err)
} else {
totalAppExecutions := statistics.TotalAppExecutions + statistics.TotalChildAppExecutions
if totalAppExecutions > int64(20000) {
org.OldOrg = true
}
}
}
// Make sure to add all orgs that are childs IF you have access
org.ChildOrgs = []OrgMini{}
wg := sync.WaitGroup{}
ch := make(chan OrgMini, len(user.Orgs))
for _, orgloop := range user.Orgs {
wg.Add(1)
// Goroutine this
go func(orgId string) {
suborg, err := GetOrg(ctx, orgId)
if err != nil {
ch <- OrgMini{}
wg.Done()
return
}
// Check if current user is in that org
found := false
for _, userloop := range suborg.Users {
if userloop.Id == user.Id {
found = true
}
}
if !found {
ch <- OrgMini{}
wg.Done()
return
}
if suborg.CreatorOrg == org.Id {
ch <- OrgMini{
Id: suborg.Id,
Name: suborg.Name,
CreatorOrg: suborg.CreatorOrg,
Image: suborg.Image,
RegionUrl: suborg.RegionUrl,
}
} else {
ch <- OrgMini{}
}
wg.Done()
}(orgloop)
}
wg.Wait()
close(ch)
for suborg := range ch {
if suborg.CreatorOrg == org.Id {
suborg.Image = ""
org.ChildOrgs = append(org.ChildOrgs, suborg)
}
}
org.SyncConfig.Apikey = ""
org.SyncConfig.Source = ""
if user.SupportAccess {
// send all suborgs for support users
org.ChildOrgs = []OrgMini{}
allChildOrgs, _, err := GetAllChildOrgs(ctx, org.Id)
if err != nil {
log.Printf("[ERROR] Failed getting child orgs for %s: %s", org.Id, err)
} else {
for _, childorg := range allChildOrgs {
// add only those that are not added to org.ChildOrgs
found := false
for _, suborg := range org.ChildOrgs {
if suborg.Id == childorg.Id {
found = true
break
}
}
if !found {
org.ChildOrgs = append(org.ChildOrgs, OrgMini{
Id: childorg.Id,
Name: childorg.Name,
CreatorOrg: childorg.CreatorOrg,
Image: childorg.Image,
RegionUrl: childorg.RegionUrl,
})
}
}
}
} else {
org.Users = []User{}
}
// This is for sending branding information
// to those who need it
if sanitizeOrg {
newOrg := org
org = &Org{}
org.Name = newOrg.Name
org.Id = newOrg.Id
org.Image = newOrg.Image
org.RegionUrl = newOrg.RegionUrl
org.Org = newOrg.Org
}
if len(org.ManagerOrgs) > 0 {
org.LeadInfo.SubOrg = true
}
if !user.SupportAccess {
org.LeadInfo = LeadInfo{}
}
newjson, err := json.Marshal(org)
if err != nil {
log.Printf("[ERROR] Failed unmarshal of org %s (%s): %s", org.Name, org.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func HandleGetSubOrgs(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting GET SUBORG request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
ctx := GetContext(request)
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get org: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
var orgId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short (get suborgs): %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
orgId = location[4]
}
if strings.Contains(orgId, "?") {
orgId = strings.Split(orgId, "?")[0]
}
org, err := GetOrg(ctx, orgId)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", orgId, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org details"}`))
return
}
userFound := false
parentUser := false // to check if the user belongs to the parent
for _, inneruser := range org.Users {
if inneruser.Id == user.Id {
userFound = true
break
}
}
parentOrg := &Org{}
isParentAdmin := false
if org.CreatorOrg != "" {
parentOrg, err = GetOrg(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[ERROR] Failed getting parent org '%s': %s", org.CreatorOrg, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting parent org details"}`))
return
}
} else {
parentOrg = org
}
for _, inneruser := range parentOrg.Users {
if inneruser.Id == user.Id {
parentUser = true
if inneruser.Role == "admin" {
isParentAdmin = true
}
break
}
}
if !userFound && !parentUser && !user.SupportAccess {
log.Printf("[ERROR] User '%s' (%s) isn't a part of org %s (%s) (get suborgs from parent)", user.Username, user.Id, parentOrg.Name, parentOrg.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User doesn't have access to org"}`))
return
}
cursor := ""
cursorList, cursorOk := request.URL.Query()["cursor"]
if cursorOk && len(cursorList) > 0 {
cursor = cursorList[0]
}
isSupportOrAdmin := user.SupportAccess || isParentAdmin
childorgs, outputCursor, err := GetAllChildOrgs(ctx, parentOrg.Id, cursor)
if err != nil || len(childorgs) == 0 {
if len(childorgs) != 0 {
log.Printf("[ERROR] Failed getting child orgs for %s. Got %d: %s", parentOrg.Id, len(childorgs), err)
}
} else {
parentOrg.ChildOrgs = []OrgMini{}
for _, childorg := range childorgs {
parentOrg.ChildOrgs = append(parentOrg.ChildOrgs, OrgMini{
Id: childorg.Id,
Name: childorg.Name,
Role: childorg.Role,
CreatorOrg: childorg.CreatorOrg,
Image: childorg.Image,
RegionUrl: childorg.RegionUrl,
})
}
}
subOrgs := []OrgMini{}
if isSupportOrAdmin {
for _, orgloop := range parentOrg.ChildOrgs {
childorg, err := GetOrg(ctx, orgloop.Id)
if err != nil {
continue
}
subOrgs = append(subOrgs, OrgMini{
Id: childorg.Id,
Name: childorg.Name,
Role: childorg.Role,
CreatorOrg: childorg.CreatorOrg,
Image: childorg.Image,
RegionUrl: childorg.RegionUrl,
})
}
} else {
for _, orgloop := range user.Orgs {
childorg, err := GetOrg(ctx, orgloop)
if err != nil {
continue
}
found := false
for _, userloop := range childorg.Users {
if userloop.Id == user.Id {
found = true
}
}
if !found {
continue
}
if childorg.CreatorOrg == org.Id {
subOrgs = append(subOrgs, OrgMini{
Id: childorg.Id,
Name: childorg.Name,
Role: childorg.Role,
CreatorOrg: childorg.CreatorOrg,
Image: childorg.Image,
RegionUrl: childorg.RegionUrl,
})
}
}
}
returnParent := OrgMini{}
//if parentOrg.CreatorOrg != "" && (parentUser || user.SupportAccess) {
returnParent = OrgMini{
Id: parentOrg.Id,
Name: parentOrg.Name,
Role: parentOrg.Role,
CreatorOrg: parentOrg.CreatorOrg,
Image: parentOrg.Image,
RegionUrl: parentOrg.RegionUrl,
}
//}
nameFilter, nameFilterOk := request.URL.Query()["name"]
if nameFilterOk && len(nameFilter) > 0 && len(nameFilter[0]) > 0 {
filteredSubOrgs := []OrgMini{}
matchingOrgs := []Org{}
for _, subOrg := range subOrgs {
if subOrg.Name == nameFilter[0] {
fullOrg, err := GetOrg(ctx, subOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting full org for %s: %s", subOrg.Id, err)
continue
}
matchingOrgs = append(matchingOrgs, *fullOrg)
}
}
if len(matchingOrgs) > 0 {
sort.Slice(matchingOrgs, func(i, j int) bool {
return matchingOrgs[i].Created > matchingOrgs[j].Created
})
latestOrg := matchingOrgs[0]
filteredSubOrgs = append(filteredSubOrgs, OrgMini{
Id: latestOrg.Id,
Name: latestOrg.Name,
Role: latestOrg.Role,
CreatorOrg: latestOrg.CreatorOrg,
Image: latestOrg.Image,
RegionUrl: latestOrg.RegionUrl,
})
}
subOrgs = filteredSubOrgs
}
data := map[string]interface{}{
"subOrgs": subOrgs,
"cursor": outputCursor,
"parentOrg": returnParent,
}
if (len(parentOrg.Id) == 0 || !parentUser) && !user.SupportAccess {
data["parentOrg"] = nil
}
finalResponse, err := json.Marshal(data)
if err != nil {
log.Printf("[ERROR] Failed to marshal JSON response: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed marshaling JSON response"}`))
return
}
resp.Header().Set("Content-Type", "application/json")
resp.WriteHeader(200)
resp.Write(finalResponse)
}
func HandleLogout(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
ctx := GetContext(request)
runReturn := false
userInfo, usererr := HandleApiAuthentication(resp, request)
log.Printf("[AUDIT] Logging out user %s (%s)", userInfo.Username, userInfo.Id)
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting LOGOUT request to main site handler (shuffler.io)")
DeleteCache(ctx, fmt.Sprintf("%s_workflows", userInfo.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("%s_workflows", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", userInfo.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", strings.ToLower(userInfo.Username)))
DeleteCache(ctx, fmt.Sprintf("user_%s", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("session_%s", userInfo.Session))
RedirectUserRequest(resp, request)
// Wait 1 second to ensure that the redirect is handled
time.Sleep(1 * time.Second)
DeleteCache(ctx, fmt.Sprintf("%s_workflows", userInfo.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("%s_workflows", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", userInfo.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", strings.ToLower(userInfo.Username)))
DeleteCache(ctx, fmt.Sprintf("user_%s", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("session_%s", userInfo.Session))
// FIXME: Allow superfluous cleanups?
// Point is: should it continue running the logout to ensure cookies are cleared?
// Keeping it for now to ensure cleanup.
return
}
}
newCookie := constructSessionDeleteCookie()
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
DeleteCache(ctx, fmt.Sprintf("%s_workflows", userInfo.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("%s_workflows", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", userInfo.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", userInfo.ActiveOrg.Id))
if runReturn == true {
DeleteCache(ctx, fmt.Sprintf("user_%s", strings.ToLower(userInfo.Username)))
DeleteCache(ctx, fmt.Sprintf("session_%s", userInfo.Session))
DeleteCache(ctx, userInfo.Session)
log.Printf("[INFO] Returning from logout request after cache cleanup")
return
}
if usererr != nil {
log.Printf("[WARNING] Api authentication failed in handleLogout: %s", usererr)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "Not logged in"}`))
return
}
DeleteCache(ctx, fmt.Sprintf("user_%s", strings.ToLower(userInfo.Username)))
DeleteCache(ctx, fmt.Sprintf("session_%s", userInfo.Session))
DeleteCache(ctx, userInfo.Session)
//store user's last session so we can force sso when user's session change.
userInfo.UsersLastSession = userInfo.Session
userInfo.Session = ""
userInfo.ValidatedSessionOrgs = []string{}
err := SetUser(ctx, &userInfo, false)
if err != nil {
log.Printf("Failed updating user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed updating apikey"}`))
return
}
ip := GetRequestIp(request)
log.Printf("[AUDIT] Successful logout for user %s (%s) from IP address %s", userInfo.Username, userInfo.Id, ip)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": false, "reason": "Successfully logged out"}`))
}
// A search for apps based on name and such
// This was before Algolia
func GetSpecificApps(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just need to be logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in set new app: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Used for searching
returnData := fmt.Sprintf(`{"success": true, "reason": []}`)
resp.WriteHeader(200)
resp.Write([]byte(returnData))
return
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type tmpStruct struct {
Search string `json:"search"`
}
var tmpBody tmpStruct
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("[WARNING] Error with unmarshal tmpBody specific apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// FIXME - continue the search here with github repos etc.
// Caching might be smart :D
ctx := GetContext(request)
workflowapps, err := GetPrioritizedApps(ctx, user)
if err != nil {
log.Printf("[WARNING] Error: Failed getting workflowapps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
returnValues := []WorkflowApp{}
search := strings.ToLower(tmpBody.Search)
for _, app := range workflowapps {
if !app.Activated && app.Generated {
// This might be heavy with A LOT
// Not too worried with todays tech tbh..
appName := strings.ToLower(app.Name)
appDesc := strings.ToLower(app.Description)
if strings.Contains(appName, search) || strings.Contains(appDesc, search) {
//log.Printf("Name: %s, Generated: %s, Activated: %s", app.Name, strconv.FormatBool(app.Generated), strconv.FormatBool(app.Activated))
returnValues = append(returnValues, app)
}
}
}
newbody, err := json.Marshal(returnValues)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow executions"}`)))
return
}
returnData = fmt.Sprintf(`{"success": true, "reason": %s}`, string(newbody))
resp.WriteHeader(200)
resp.Write([]byte(returnData))
}
func GetAppAuthentication(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in get app auth: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
allAuths, err := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get all app auth: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(allAuths) == 0 {
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "data": []}`))
return
}
// Cleanup for frontend
newAuth := []AppAuthenticationStorage{}
for _, auth := range allAuths {
newAuthField := auth
for index, _ := range auth.Fields {
newAuthField.Fields[index].Value = "Secret. Replaced during app execution!"
}
newAuth = append(newAuth, newAuthField)
}
type returnStruct struct {
Success bool `json:"success"`
Data []AppAuthenticationStorage `json:"data"`
}
allAuth := returnStruct{
Success: true,
Data: allAuths,
}
newbody, err := json.Marshal(allAuth)
if err != nil {
log.Printf("Failed unmarshalling all app auths: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow app auth"}`)))
return
}
//data := fmt.Sprintf(`{"success": true, "data": %s}`, string(newbody))
resp.WriteHeader(200)
resp.Write([]byte(newbody))
}
func AddAppAuthentication(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
if session, err := request.Cookie("__session"); err == nil {
//log.Printf("\n\n[DEBUG]: Found session token that failed. Should search org auth for %#v", session)
// Gets a sample user to use
ctx := GetContext(request)
user, err = GetOrgAuth(ctx, session.Value)
if err != nil {
log.Printf("[WARNING] Failed getting org auth for session: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if user.Id == "" || user.Role != "admin" {
log.Printf("[WARNING] Api authentication failed in add app auth: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to set new workflowapp: %s (%s)", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Error with body read in new app auth: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
var appAuth AppAuthenticationStorage
err = json.Unmarshal(body, &appAuth)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling (appauth): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[AUDIT] Setting new app authentication for app %s with user %s (%s) in org %s (%s)", appAuth.App.Name, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
ctx := GetContext(request)
org := &Org{}
originalAuth := &AppAuthenticationStorage{}
originalId := appAuth.Id
if len(appAuth.Id) == 0 {
// To not override, we should use an md5 based on the input fields + org to create the ID
fielddata := fmt.Sprintf("%s_%s", user.ActiveOrg.Id, appAuth.Label)
for _, field := range appAuth.Fields {
fielddata += field.Key
fielddata += field.Value
}
// Happens in very rare circumstances
hasher := md5.New()
hasher.Write([]byte(fielddata))
appAuth.Id = hex.EncodeToString(hasher.Sum(nil))
} else {
originalAuth, err = GetWorkflowAppAuthDatastore(ctx, appAuth.Id)
if err == nil {
// OrgId string `json:"org_id" datastore:"org_id"`
if originalAuth.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s (%s) isn't a part of the right org during auth edit", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`)))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User %s (%s) isn't admin during auth edit", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": ":("}`)))
return
}
if !originalAuth.Active {
// Forcing it active
appAuth.Active = true
/*
log.Printf("[WARNING] Auth isn't active for edit")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update an inactive auth"}`)))
return
*/
}
if originalAuth.App.Name != appAuth.App.Name {
log.Printf("[AUDIT] User %s (%s) tried to modify auth, but appname was wrong", user.Username, user.Id)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad app configuration: need to specify correct name"}`)))
return
}
//if appAuth.Type != "oauth2" && appAuth.Type != "oauth" && appAuth.Type != "oauth2-app" {
for fieldIndex, field := range appAuth.Fields {
if !strings.Contains(field.Value, "Secret. Replaced") {
continue
}
for _, existingField := range originalAuth.Fields {
if existingField.Key != field.Key {
continue
}
appAuth.Fields[fieldIndex].Value = existingField.Value
if originalAuth.Encrypted {
// Decrypt it here
parsedKey := fmt.Sprintf("%s_%d_%s_%s", originalAuth.OrgId, originalAuth.Created, originalAuth.Label, field.Key)
newValue, err := HandleKeyDecryption([]byte(existingField.Value), parsedKey)
if err != nil {
log.Printf("[WARNING] Failed decrypting field %s: %s", field.Key, err)
} else {
//log.Printf("Decrypted value: %s", newValue)
appAuth.Fields[fieldIndex].Value = string(newValue)
}
}
break
}
}
if len(appAuth.Fields) == 0 {
appAuth.Fields = originalAuth.Fields
}
// Decrypt with old label to ensure re-encryption with new label
for fieldIndex, field := range appAuth.Fields {
if len(field.Value) == 0 || strings.Contains(field.Value, "Secret. Replaced") {
for _, existingField := range originalAuth.Fields {
if existingField.Key != field.Key {
continue
}
//log.Printf("Replacing field %s with value '%s'", field.Key, existingField.Value)
// Decrypt it based on auth
parsedKey := fmt.Sprintf("%s_%d_%s_%s", originalAuth.OrgId, originalAuth.Created, originalAuth.Label, field.Key)
newValue, err := HandleKeyDecryption([]byte(existingField.Value), parsedKey)
if err != nil {
log.Printf("[WARNING] Failed decrypting field %s: %s", field.Key, err)
} else {
//log.Printf("Decrypted value: %s", newValue)
appAuth.Fields[fieldIndex].Value = string(newValue)
field.Value = string(newValue)
}
}
}
//log.Printf("Default value: %s", field.Value)
parsedKey := fmt.Sprintf("%s_%d_%s_%s", originalAuth.OrgId, originalAuth.Created, originalAuth.Label, field.Key)
newValue, err := HandleKeyDecryption([]byte(field.Value), parsedKey)
if err != nil {
log.Printf("[WARNING] Failed decrypting field %s: %s", field.Key, err)
} else {
//log.Printf("Decrypted value: %s", newValue)
appAuth.Fields[fieldIndex].Value = string(newValue)
}
}
// Setting this to ensure that any new config is encrypted anew
appAuth.Encrypted = false
//} else {
//}
} else {
// ID sometimes used in creation as well
//log.Printf("[WARNING] Failed finding existing auth: %s", err)
//resp.WriteHeader(409)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't find existing auth"}`)))
//return
}
}
if len(appAuth.Label) == 0 {
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Label can't be empty"}`)))
return
}
// Super basic check
if len(appAuth.App.ID) != 36 && len(appAuth.App.ID) != 32 {
log.Printf("[WARNING] Bad ID for app: %s", appAuth.App.ID)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "App has to be defined"}`)))
return
}
app, err := GetApp(ctx, appAuth.App.ID, user, false)
if err != nil {
log.Printf("[DEBUG] Failed finding app %s (%s) while setting auth. Finding it by looping apps.", appAuth.App.Name, appAuth.App.ID)
workflowapps, err := GetPrioritizedApps(ctx, user)
if err != nil {
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
foundIndex := -1
for i, workflowapp := range workflowapps {
if workflowapp.Name == appAuth.App.Name {
foundIndex = i
break
}
}
if foundIndex >= 0 {
log.Printf("[INFO] Found app %s (%s) by looping auth with %d parameters", workflowapps[foundIndex].Name, workflowapps[foundIndex].ID, len(workflowapps[foundIndex].Authentication.Parameters))
app = &workflowapps[foundIndex]
//appAuth.App.Name, appAuth.App.ID, len(appAuth.Fields)))
} else {
log.Printf("[ERROR] Failed finding app %s which has auth after looping", appAuth.App.ID)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed finding app %s (%s)"}`, appAuth.App.Name, appAuth.App.ID)))
return
}
} else {
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org %s during app auth: %s", user.ActiveOrg.Id, err)
} else {
if !ArrayContains(org.ActiveApps, app.ID) {
org.ActiveApps = append(org.ActiveApps, app.ID)
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed setting app %s for org %s during appauth", app.ID, org.Id)
} else {
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-100"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-500"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-1000"))
DeleteCache(ctx, "all_apps")
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
}
} else {
log.Printf("[INFO] Org %s (%s) already has app %s active.", user.ActiveOrg.Name, user.ActiveOrg.Id, app.ID)
}
}
}
// Only in this one if NEW oauth2 auth
if appAuth.Type == "oauth2" && len(originalId) == 0 {
log.Printf("[DEBUG] OAUTH2 for workflow %s. User: %s (%s)", appAuth.ReferenceWorkflow, user.Username, user.Id)
if len(appAuth.ReferenceWorkflow) > 0 {
workflow, err := GetWorkflow(ctx, appAuth.ReferenceWorkflow)
if err != nil {
log.Printf("[WARNING] WorkflowId %s doesn't exist (set oauth2)", appAuth.ReferenceWorkflow)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
if workflow.OrgId == user.ActiveOrg.Id && user.Role != "org-reader" {
log.Printf("[AUDIT] User %s is accessing workflow '%s' as admin (set oauth2)", user.Username, workflow.ID)
} else if workflow.Public {
log.Printf("[AUDIT] Letting user %s access workflow %s FOR AUTH because it's public", user.Username, workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (set oauth2)", user.Username, workflow.ID)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Your user is not allowed to set authentication for this workflow in Shuffle."}`))
return
}
}
// Finding count in same workflow & setting large image if missing
count := 0
for actionIndex, action := range workflow.Actions {
if action.AppName != appAuth.App.Name {
continue
}
count += 1
workflow.Actions[actionIndex].AuthenticationId = appAuth.Id
if len(appAuth.App.LargeImage) == 0 && len(action.LargeImage) > 0 {
appAuth.App.LargeImage = action.LargeImage
}
}
if count > 0 {
err = SetWorkflow(ctx, *workflow, workflow.ID)
if err != nil {
log.Printf("[WARNING] Failed setting workflow %s during oauth2 auth update: %s", workflow.ID, err)
} else {
log.Printf("[INFO] Updated %d actions in workflow %s with auth %s from Oauth2", count, workflow.ID, appAuth.Id)
}
}
appAuth.NodeCount = int64(count)
appAuth.WorkflowCount = 1
}
_, err = RunOauth2Request(ctx, user, appAuth, false)
if err != nil {
parsederror := strings.Replace(fmt.Sprintf("%s", err), "\"", "\\\"", -1)
log.Printf("[WARNING] Failed oauth2 request (3): %s", err)
if strings.Contains(fmt.Sprintf("%s", err), "not consented") {
log.Printf("Return the user to the URL with admin consent")
}
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed authorization: %s"}`, parsederror)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully set up authentication", "id": "%s"}`, appAuth.Id)))
return
} else if appAuth.Type == "oauth2-app" && len(originalId) == 0 {
// For application permissions set in Oauth2 frontend
// This should contain client-id, client-secret, scopes, token-url
// May need to also know how the auth actually works (e.g. basic auth or something else)
appAuth.App.AppVersion = app.AppVersion
log.Printf("[DEBUG] OAUTH2-APP for workflow %s. User: %s (%s). App: %s (%s)", appAuth.ReferenceWorkflow, user.Username, user.Id, appAuth.App.Name, appAuth.App.ID)
// Testing if the auth works
_, err := GetOauth2ApplicationPermissionToken(ctx, user, appAuth)
if err != nil {
log.Printf("\n[WARNING] Failed getting oauth2 application permission token: %s\n\n", err)
resp.WriteHeader(400)
parsedOutput := ResultChecker{
Success: false,
Reason: fmt.Sprintf("Failed auth. Is your Client ID, Client Secret and Scopes correct?\n\nError: %s", err),
}
marshalledOutput, err := json.Marshal(parsedOutput)
if err != nil {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed authorization. Is your client ID, client secret and scope correct? Raw: %s"}`, strings.Replace(err.Error(), "\"", "\\\"", -1))))
return
}
resp.Write(marshalledOutput)
return
}
} else {
// Edgecases for oauth2 as they need reauth
if appAuth.Type == "oauth2" || appAuth.Type == "oauth2-app" {
for _, field := range appAuth.Fields {
if field.Key != "url" {
continue
}
if len(field.Value) == 0 {
log.Printf("[WARNING] Failed finding field 'url' in appauth fields for %s", appAuth.App.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Field can't be empty: url"}`))
return
}
// Trim
field.Value = strings.TrimSpace(field.Value)
// Valid: https://example.com
// Invalid: http://example.com/
// Invalid: example.com
if !strings.HasPrefix(field.Value, "http") || strings.HasSuffix(field.Value, "/") || !strings.Contains(field.Value, "://") || strings.Contains(field.Value, " ") || strings.Contains(field.Value, "\n") {
log.Printf("[WARNING] Invalid URL for field 'url' in appauth edit: %#v", field.Value)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Field must be a valid URL, and NOT end with /"}`))
return
}
found := false
for paramIndex, param := range originalAuth.Fields {
if param.Key != field.Key {
continue
}
// Encrypt the url field?
// Skipping for now as it's not as sensitive a field, and we may even add editing possibilities to it in the future.
found = true
log.Printf("[DEBUG] Replacing URL field in appauth for %s from %#v to %#v", appAuth.App.Name, originalAuth.Fields[paramIndex].Value, field.Value)
originalAuth.Fields[paramIndex].Value = field.Value
}
if !found {
log.Printf("[WARNING] Failed finding field '%s' in OAUTH2 appauth fields for %s", field.Key, appAuth.App.Name)
continue
}
}
appAuth.Fields = originalAuth.Fields
} else {
// Check if the items are correct
for _, field := range appAuth.Fields {
found := false
for _, param := range app.Authentication.Parameters {
//log.Printf("Fields: %s - %s", field, param.Name)
if field.Key == param.Name {
found = true
}
}
if !found {
log.Printf("[WARNING] Failed finding field '%s' in appauth fields for %s", field.Key, appAuth.App.Name)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "All auth fields required"}`)))
return
}
}
}
}
if len(appAuth.App.LargeImage) == 0 && len(app.LargeImage) > 0 {
appAuth.App.LargeImage = app.LargeImage
}
// If manual editing, reset verification
appAuth.Validation = TypeValidation{}
appAuth.OrgId = user.ActiveOrg.Id
appAuth.Defined = true
err = SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id)
if err != nil {
log.Printf("[WARNING] Failed setting up app auth %s: %s", appAuth.Id, err)
resp.WriteHeader(409)
resultData := ResultChecker{
Success: false,
Reason: fmt.Sprintf("%s", err),
}
newjson, err := json.Marshal(resultData)
if err != nil {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
} else {
resp.Write(newjson)
}
return
}
if appAuth.AutoDistribute {
log.Printf("[DEBUG] Auto distributing auth %s for app %s (%s) in org %s (%s) to all workflows.", appAuth.Id, appAuth.App.Name, appAuth.App.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
err := AssignAuthEverywhere(ctx, &appAuth, user)
if err != nil {
log.Printf("[ERROR] Failed assigning auth everywhere (2): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting workflows to update"}`))
} else {
log.Printf("[INFO] Assigned auth everywhere")
}
}
// Set it as the default app in the org as it's the "latest" of it's kind.
// This just means adding it to the app framework if a category exists
if len(app.Categories) > 0 {
// Get org
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err == nil {
lowercased := strings.ToLower(app.Categories[0])
if lowercased == "communication" || lowercased == "email" {
org.SecurityFramework.Communication.ID = app.ID
org.SecurityFramework.Communication.Name = app.Name
org.SecurityFramework.Communication.LargeImage = app.LargeImage
org.SecurityFramework.Communication.Description = app.Description
} else if lowercased == "siem" {
org.SecurityFramework.SIEM.ID = app.ID
org.SecurityFramework.SIEM.Name = app.Name
org.SecurityFramework.SIEM.LargeImage = app.LargeImage
org.SecurityFramework.SIEM.Description = app.Description
} else if lowercased == "assets" {
org.SecurityFramework.Assets.ID = app.ID
org.SecurityFramework.Assets.Name = app.Name
org.SecurityFramework.Assets.LargeImage = app.LargeImage
org.SecurityFramework.Assets.Description = app.Description
} else if lowercased == "cases" {
org.SecurityFramework.Cases.ID = app.ID
org.SecurityFramework.Cases.Name = app.Name
org.SecurityFramework.Cases.LargeImage = app.LargeImage
org.SecurityFramework.Cases.Description = app.Description
} else if lowercased == "network" {
org.SecurityFramework.Network.ID = app.ID
org.SecurityFramework.Network.Name = app.Name
org.SecurityFramework.Network.LargeImage = app.LargeImage
org.SecurityFramework.Network.Description = app.Description
} else if lowercased == "intel" {
org.SecurityFramework.Intel.ID = app.ID
org.SecurityFramework.Intel.Name = app.Name
org.SecurityFramework.Intel.LargeImage = app.LargeImage
org.SecurityFramework.Intel.Description = app.Description
} else if lowercased == "edr" {
org.SecurityFramework.EDR.ID = app.ID
org.SecurityFramework.EDR.Name = app.Name
org.SecurityFramework.EDR.LargeImage = app.LargeImage
org.SecurityFramework.EDR.Description = app.Description
} else if lowercased == "iam" {
org.SecurityFramework.IAM.ID = app.ID
org.SecurityFramework.IAM.Name = app.Name
org.SecurityFramework.IAM.LargeImage = app.LargeImage
org.SecurityFramework.IAM.Description = app.Description
} else if lowercased == "ai" {
org.SecurityFramework.AI.ID = app.ID
org.SecurityFramework.AI.Name = app.Name
org.SecurityFramework.AI.LargeImage = app.LargeImage
org.SecurityFramework.AI.Description = app.Description
} else {
log.Printf("[ERROR] Unknown category %s for app %s (%s)", lowercased, app.Name, app.ID)
}
// Set the org
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed setting org after setting default app: %s", err)
}
}
}
if appAuth.SuborgDistributed {
// Clear auth cache for all suborgs
//nameKey := "workflowappauth"
//cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)
if len(org.Id) == 0 {
org, err = GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org for suborg auth clear: %s", err)
}
}
for _, childOrg := range org.ChildOrgs {
cacheKey := fmt.Sprintf("workflowappauth_%s", childOrg.Id)
DeleteCache(ctx, cacheKey)
}
}
log.Printf("[INFO] Set new app auth for %s (%s) with ID %s", app.Name, app.ID, appAuth.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, appAuth.Id)))
}
func AddAppAuthenticationGroup(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in add app auth group: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Need to be admin to add appauth group")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Error with body read in new app auth group: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
var appAuthGroup AppAuthenticationGroup
if err := json.Unmarshal(body, &appAuthGroup); err != nil {
log.Printf("[WARNING] Failed unmarshaling (appauthgroup): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
if len(appAuthGroup.Id) > 0 {
// Get group and check if it's in the org
authGroup, err := GetAppAuthGroup(ctx, appAuthGroup.Id)
if err != nil {
log.Printf("[WARNING] Failed finding app auth group %s: %s", appAuthGroup.Id, err)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't find existing app auth group"}`)))
return
}
if authGroup.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s (%s) isn't a part of the right org during auth group edit", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No access"}`)))
return
}
}
log.Printf("[AUDIT] Setting new app authentication group for %s with user %s (%s) in org %s (%s)", appAuthGroup.Label, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
if len(appAuthGroup.Label) == 0 {
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Label can't be empty"}`)))
return
}
// Super basic check
if len(appAuthGroup.AppAuths) == 0 {
log.Printf("[WARNING] Empty appauths for appauthgroup")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Apps have to be defined"}`)))
return
}
appAuthGroup.OrgId = user.ActiveOrg.Id
appAuthGroup.Active = true
if len(appAuthGroup.Id) == 0 {
appAuthGroup.Id = uuid.NewV4().String()
}
err = SetAuthGroupDatastore(ctx, appAuthGroup, appAuthGroup.Id)
if err != nil {
log.Printf("[WARNING] Failed setting up app auth group %s: %s", appAuthGroup.Id, err)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, appAuthGroup.Id)))
}
func GetAppAuthenticationGroup(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in get app auth group: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
allAuthGroups, err := GetAuthGroups(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get all app auth group: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(allAuthGroups) == 0 {
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "data": []}`))
return
}
// Cleanup for frontend
newAuthGroups := []AppAuthenticationGroup{}
for _, authGroup := range allAuthGroups {
newAuthGroup := authGroup
newAuthGroups = append(newAuthGroups, newAuthGroup)
}
type returnStruct struct {
Success bool `json:"success"`
Data []AppAuthenticationGroup `json:"data"`
}
allAuth := returnStruct{
Success: true,
Data: allAuthGroups,
}
newbody, err := json.Marshal(allAuth)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling all app auth groups: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow app auth group"}`)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(newbody))
}
func DeleteAppAuthenticationGroup(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in delete app auth: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Need to be admin to delete appauth")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 5 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[5]
}
log.Printf("[AUDIT] Deleting app auth group %s with user %s (%s) in org %s (%s)", fileId, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
ctx := GetContext(request)
nameKey := "workflowappauthgroup"
auth, err := GetAppAuthGroup(ctx, fileId)
if err != nil {
// Deleting cache here, as it seems to be a constant issue
cacheKey := fmt.Sprintf("%s_%s", nameKey, user.ActiveOrg.Id)
DeleteCache(ctx, cacheKey)
log.Printf("[WARNING] Authget group error (DELETE): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": ":("}`))
return
}
if auth.OrgId != user.ActiveOrg.Id {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User can't edit this org"}`))
return
}
// FIXME: Set affected workflows to have errors
// 1. Get the auth
// 2. Loop the workflows (.Usage) and set them to have errors
// 3. Loop the nodes in workflows and do the same
err = DeleteKey(ctx, nameKey, fileId)
if err != nil {
log.Printf("Failed deleting workflowapp")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`)))
return
}
cacheKey := fmt.Sprintf("%s_%s", nameKey, user.ActiveOrg.Id)
DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("%s_%s", nameKey, fileId)
DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func DeleteAppAuthentication(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in delete app auth: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Need to be admin to delete appauth")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 5 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[5]
}
ctx := GetContext(request)
nameKey := "workflowappauth"
auth, err := GetWorkflowAppAuthDatastore(ctx, fileId)
if err != nil {
// Deleting cache here, as it seems to be a constant issue
cacheKey := fmt.Sprintf("%s_%s", nameKey, user.ActiveOrg.Id)
DeleteCache(ctx, cacheKey)
log.Printf("[WARNING] Authget error (DELETE): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": ":("}`))
return
}
if auth.OrgId != user.ActiveOrg.Id {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User can't edit this org"}`))
return
}
// FIXME: Set affected workflows to have errors
// 1. Get the auth
// 2. Loop the workflows (.Usage) and set them to have errors
// 3. Loop the nodes in workflows and do the same
err = DeleteKey(ctx, nameKey, fileId)
if err != nil {
log.Printf("Failed deleting workflowapp")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting workflow app"}`)))
return
}
cacheKey := fmt.Sprintf("%s_%s", nameKey, user.ActiveOrg.Id)
DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("%s_%s", nameKey, fileId)
DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func HandleSetEnvironments(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Only admin can change environments, but if there are no users, anyone can make (first)
user, err := HandleApiAuthentication(resp, request)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't set environment without being admin"}`))
return
}
ctx := GetContext(request)
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting environments: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't get environments when setting"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading environment body: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`)))
return
}
var newEnvironments []Environment
err = json.Unmarshal(body, &newEnvironments)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`)))
return
}
log.Printf("[WARNING] Got %d new environments to be added", len(newEnvironments))
if len(newEnvironments) < 1 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "One environment is required"}`)))
return
}
if project.Environment == "cloud" {
//foundOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
//if err != nil {
// resp.WriteHeader(401)
// resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`)))
// return
//}
// FIXME: Removed need for syncfeatures to be enabled
// September 2022
//_ = foundOrg
//if !foundOrg.SyncFeatures.MultiEnv.Active {
// resp.WriteHeader(401)
// resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Adding multiple environments requires an active hybrid, enterprise or MSSP subscription"}`)))
// return
//}
}
if project.Environment == "onprem" {
currentOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`)))
return
}
parentOrg := currentOrg
if len(currentOrg.CreatorOrg) > 0 {
parentOrg, err = GetOrg(ctx, currentOrg.CreatorOrg)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`)))
return
}
}
envs, err := GetEnvironments(ctx, currentOrg.Id)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to get environments of organization"}`)))
return
}
licenseOrg := HandleCheckLicense(ctx, *parentOrg)
if int64(len(envs)) > licenseOrg.SyncFeatures.MultiEnv.Limit {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You have reached the limit of %d environments for your subscription. Upgrade to an enterprise plan or contact support@shuffler.io for more info."}`, licenseOrg.SyncFeatures.MultiEnv.Limit)))
return
}
}
// Validate input here
defaults := 0
parsedEnvs := []Environment{}
for _, env := range newEnvironments {
if project.Environment == "cloud" && env.Type == "cloud" && env.Archived {
log.Printf("[WARNING] User %s (%s) tried to disable the cloud environment", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't disable cloud environments"}`))
return
}
if env.Default && env.Archived {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't disable default environment"}`))
return
}
//if project.Environment == "cloud" && env.Type != "cloud" && len(env.Name) < 10 {
// log.Printf("[ERROR] Skipping env %s because length is shorter than 10", env.Name)
// continue
//}
if defaults > 0 {
env.Default = false
}
if env.Default {
defaults += 1
}
parsedEnvs = append(parsedEnvs, env)
}
newEnvironments = parsedEnvs
openEnvironments := 0
for _, item := range newEnvironments {
if !item.Archived {
openEnvironments += 1
}
}
if openEnvironments < 1 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't archive all environments. Not deleting."}`))
return
}
// Clear old data? Removed for archiving purpose. No straight deletion
log.Printf("[INFO] Deleting %d original environments before resetting. To be added: %d!", len(environments), len(newEnvironments))
nameKey := "Environments"
for _, item := range environments {
DeleteKey(ctx, nameKey, item.Id)
DeleteKey(ctx, nameKey, item.Name)
}
for _, item := range newEnvironments {
for _, subenv := range environments {
if item.Name == subenv.Name || item.Id == subenv.Id {
item.Auth = subenv.Auth
break
}
}
item.RunningIp = ""
if item.OrgId != user.ActiveOrg.Id && len(item.SuborgDistribution) == 0 {
item.OrgId = user.ActiveOrg.Id
}
if len(item.Id) == 0 {
item.Id = uuid.NewV4().String()
}
if len(item.Auth) == 0 {
item.Auth = uuid.NewV4().String()
}
err = SetEnvironment(ctx, &item)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed setting environment variable"}`))
return
}
}
cacheKey := fmt.Sprintf("Environments_%s", user.ActiveOrg.Id)
DeleteCache(ctx, cacheKey)
log.Printf("[INFO] Set %d new environments for org %s", len(newEnvironments), user.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func RerunExecution(ctx context.Context, environment string, workflow Workflow) (int, error) {
maxReruns := 100
executions, err := GetUnfinishedExecutions(ctx, workflow.ID)
if err != nil {
log.Printf("[DEBUG] Failed getting executions for workflow %s", workflow.ID)
return 0, err
}
if len(executions) == 0 {
return 0, nil
}
//if project.Environment == "cloud" {
// backendUrl = "https://shuffler.io"
//} else {
// backendUrl = "http://127.0.0.1:5001"
//}
//topClient := &http.Client{
// Transport: &http.Transport{
// Proxy: nil,
// },
//}
//_ = backendUrl
//_ = topClient
//StartedAt int64 `json:"started_at" datastore:"started_at"`
timeNow := int64(time.Now().Unix())
cnt := 0
// Rerun after 570 seconds (9.5 minutes), ensuring it can check 3 times before
// automated aborting of the execution happens
waitTime := 270
//waitTime := 0
executed := []string{}
for _, execution := range executions {
if timeNow < execution.StartedAt+int64(waitTime) {
continue
}
if execution.Status != "EXECUTING" {
continue
}
if ArrayContains(executed, execution.ExecutionId) {
continue
}
executed = append(executed, execution.ExecutionId)
found := false
environments := []string{}
for _, action := range execution.Workflow.Actions {
if action.Environment == environment {
environments = append(environments, action.Environment)
found = true
break
}
}
if len(environments) == 0 {
found = true
}
if !found {
continue
}
if cnt > maxReruns {
log.Printf("[DEBUG] Breaking because more than 100 executions are executing")
break
}
if project.Environment != "cloud" {
executionRequest := ExecutionRequest{
ExecutionId: execution.ExecutionId,
WorkflowId: execution.Workflow.ID,
Authorization: execution.Authorization,
Environments: environments,
}
executionRequest.Priority = execution.Priority
err = SetWorkflowQueue(ctx, executionRequest, environment)
if err != nil {
log.Printf("[ERROR] Failed re-adding execution to db: %s", err)
}
} else {
//log.Printf("[DEBUG] Rerunning executions is not available in cloud yet.")
//if len(environments) != 1 || strings.ToLower(environments[0]) != "cloud" {
// log.Printf("[DEBUG][%s] Skipping execution for workflow %s because it's not for JUST the cloud env. Org: %s", execution.ExecutionId, execution.Workflow.ID, execution.OrgId)
// continue
//}
streamUrl := fmt.Sprintf("https://shuffler.io")
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
streamUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
streamUrl = fmt.Sprintf("%s", os.Getenv("SHUFFLE_CLOUDRUN_URL"))
}
streamUrl = fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/rerun", streamUrl, execution.Workflow.ID, execution.ExecutionId)
client := &http.Client{
Timeout: 5 * time.Second,
}
req, err := http.NewRequest(
"POST",
streamUrl,
nil,
)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", execution.Authorization))
if err != nil {
log.Printf("[WARNING] Error in new request for manual rerun: %s", err)
continue
}
newresp, err := client.Do(req)
if err != nil {
log.Printf("[WARNING] Error running body for manual rerun: %s", err)
continue
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body for manual rerun: %s", err)
continue
}
log.Printf("[DEBUG] Rerun response: %s", string(body))
}
cnt += 1
log.Printf("[DEBUG] Should rerun execution %s (%s - Workflow: %s) with environments %s", execution.ExecutionId, execution.Status, execution.Workflow.ID, environments)
//log.Printf("[DEBUG] Result from rerunning %s: %s", execution.ExecutionId, string(body))
}
return cnt, nil
}
func CleanupExecutions(ctx context.Context, environment string, workflow Workflow, cleanAll bool) (int, error) {
executions, err := GetUnfinishedExecutions(ctx, workflow.ID)
if err != nil {
log.Printf("[DEBUG] Failed getting executions for workflow %s", workflow.ID)
return 0, err
}
if len(executions) == 0 {
return 0, nil
}
//log.Printf("[DEBUG] Found %d POTENTIALLY unfinished executions for workflow %s (%s) with environment %s that are more than 30 minutes old", len(executions), workflow.Name, workflow.ID, environment)
//log.Printf("[DEBUG] Found %d unfinished executions for workflow %s (%s) with environment %s that are more than 30 minutes old", len(executions), workflow.Name, workflow.ID, environment)
backendUrl := os.Getenv("BASE_URL")
// Redundant, but working ;)
if project.Environment == "cloud" {
backendUrl = "https://shuffler.io"
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
} else {
backendUrl = "http://127.0.0.1:5001"
}
topClient := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
//StartedAt int64 `json:"started_at" datastore:"started_at"`
timeNow := int64(time.Now().Unix())
cnt := 0
for _, execution := range executions {
if cleanAll {
} else if timeNow < execution.StartedAt+1800 {
//log.Printf("Bad timing: %d", execution.StartedAt)
continue
}
if execution.Status != "EXECUTING" {
//log.Printf("[ERROR][%s] Bad status for execution: %s", execution.ExecutionId, execution.Status)
continue
}
found := false
environments := []string{}
for _, action := range execution.Workflow.Actions {
if action.Environment == environment {
environments = append(environments, action.Environment)
found = true
break
}
}
if len(environments) == 0 {
found = true
}
if !found {
continue
}
streamUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort?reason=%s", backendUrl, execution.Workflow.ID, execution.ExecutionId, url.QueryEscape(`{"success": False, "reason": "Shuffle's automated cleanup bot stopped this execution as it didn't finish within 30 minutes.", "details": "You may disable this by setting this environment variable on your backend container: SHUFFLE_DISABLE_RERUN_AND_ABORT=true"}`))
//log.Printf("Url: %s", streamUrl)
req, err := http.NewRequest(
"GET",
streamUrl,
nil,
)
if err != nil {
log.Printf("[ERROR] Error in auto-abort request: %s", err)
continue
}
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, execution.Authorization))
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Error auto-aborting workflow: %s", err)
continue
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading parent body: %s", err)
continue
}
//log.Printf("BODY (%d): %s", newresp.StatusCode, string(body))
if newresp.StatusCode != 200 {
log.Printf("[ERROR] Bad statuscode in auto-abort: %d, %s", newresp.StatusCode, string(body))
continue
}
cnt += 1
if debug {
log.Printf("[DEBUG] Result from aborting %s: %s", execution.ExecutionId, string(body))
}
}
return cnt, nil
}
func HandleGetEnvironments(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get environments: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting environments: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Can't get environments"}`))
return
}
// Check if there is any parameter for specific environments
var findEnv string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
//log.Printf("[ERROR] Path too short (get environments): %d", len(location))
} else {
findEnv = strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(location[4], "%20", "_"), " ", "_"))
}
}
if len(findEnv) > 0 {
newEnvironments := []Environment{}
for _, env := range environments {
parsedName := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(env.Name, "%20", "_"), " ", "_"))
if parsedName == findEnv || env.Id == findEnv {
newEnvironments = append(newEnvironments, env)
break
}
}
environments = newEnvironments
if len(environments) == 0 {
resp.WriteHeader(404)
resp.Write([]byte(`{"success": false, "reason": "Can't find environment. Does it exist?"}`))
return
}
}
// Always make Cloud the default environment
// If there are multiple and none are chosen
if project.Environment == "cloud" && findEnv == "" {
defaults := []int{}
cloudFound := false
for envIndex, environment := range environments {
if environment.Default {
defaults = append(defaults, envIndex)
}
if strings.ToLower(environment.Name) == "cloud" {
cloudFound = true
}
}
// Ensure it's attached. When they click "set as default", it will become activated forever :>
// Found by seeing a user from early on that didn't have the env
if !cloudFound {
setDefault := false
if len(environments) == 1 || len(defaults) == 0 {
setDefault = true
}
environments = append(environments, Environment{
Name: "Cloud",
Type: "cloud",
Archived: false,
Registered: true,
Default: setDefault,
OrgId: user.ActiveOrg.Id,
Id: uuid.NewV4().String(),
})
defaults = append(defaults, len(environments)-1)
}
// Fallback to cloud for now
if len(defaults) > 1 {
for _, index := range defaults {
if strings.ToLower(environments[index].Name) == "cloud" {
continue
} else {
environments[index].Default = false
}
}
}
}
hideEnvs := false
parentOrgMain := Org{}
if project.Environment == "onprem" && findEnv == "" {
currentOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`)))
return
}
parentOrg := currentOrg
if len(currentOrg.CreatorOrg) > 0 {
parentOrg, err = GetOrg(ctx, currentOrg.CreatorOrg)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed find your organization"}`)))
return
}
}
licenseOrg := HandleCheckLicense(ctx, *parentOrg)
parentOrgMain = licenseOrg
if int64(len(environments)) > licenseOrg.SyncFeatures.MultiEnv.Limit {
hideEnvs = true
}
}
newEnvironments := []Environment{}
sort.Slice(environments, func(i, j int) bool {
return environments[i].Created < environments[j].Created
})
filteredEnvironments := []Environment{}
if findEnv != "" {
newEnvironments = environments
} else if hideEnvs {
defaultEnvs := []Environment{}
nonDefaultEnvs := []Environment{}
for _, env := range environments {
if len(env.Id) == 0 {
env.Id = uuid.NewV4().String()
}
if env.Default {
defaultEnvs = append(defaultEnvs, env)
} else {
nonDefaultEnvs = append(nonDefaultEnvs, env)
}
}
filteredEnvironments = append(filteredEnvironments, defaultEnvs...)
limit := int(parentOrgMain.SyncFeatures.MultiEnv.Limit)
for i, env := range nonDefaultEnvs {
if i < (limit - 1) {
filteredEnvironments = append(filteredEnvironments, env)
}
}
environments = filteredEnvironments
} else {
for i := range environments {
if len(environments[i].Id) == 0 {
environments[i].Id = uuid.NewV4().String()
}
}
}
for _, environment := range environments {
found := false
for _, oldEnv := range newEnvironments {
if oldEnv.Name == environment.Name {
found = true
}
}
if !found {
// Get the current Queue for it
environment.Queue = -1
if len(environments) < 10 && environment.Type != "cloud" {
//log.Printf("\n\nShould get queue for env %s (%s)\n\n", environment.Name, environment.Id)
//executionRequests, err := GetWorkflowQueue(ctx, environment.Id, 100)
foundName := environment.Name
if project.Environment == "cloud" {
foundName = fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(environment.Name, " ", "-"), "_", "-")), user.ActiveOrg.Id)
}
executionRequests, err := GetWorkflowQueue(ctx, foundName, 100)
if err != nil {
// Skipping as this comes up over and over
log.Printf("[ERROR] (2) Failed reading body for workflowqueue: %s", err)
} else {
environment.Queue = len(executionRequests.Data)
}
//log.Printf("[DEBUG] Got %d executions for env %s", len(executionRequests.Data), environment.Name)
}
newEnvironments = append(newEnvironments, environment)
}
}
// Resets ips and such very quickly using cache
// Here as well as in db-connector due to cache handling
timenow := time.Now().Unix()
for envIndex, env := range newEnvironments {
if newEnvironments[envIndex].Type != "onprem" {
continue
}
if newEnvironments[envIndex].Archived {
continue
}
// Check for env updates from cache just in case to keep things up to date
// The timeout for this key is 2 minutes, meaning we very quickly get the right answer/timeouts
cacheKey := fmt.Sprintf("queueconfig-%s-%s", env.Name, env.OrgId)
cache, err := GetCache(ctx, cacheKey)
if err == nil {
newEnv := OrborusStats{}
err = json.Unmarshal(cache.([]uint8), &newEnv)
if err == nil {
// Check if timestamp is within the last 180 seconds. If it is, overwrite newEnvironments
if newEnv.Timestamp > 0 && timenow-newEnv.Timestamp > 180 {
newEnvironments[envIndex].RunningIp = ""
//newEnvironments[envIndex].Licensed = false
newEnvironments[envIndex].DataLake.Enabled = false
} else {
newEnvironments[envIndex].DataLake = newEnv.DataLake
newEnvironments[envIndex].RunningIp = newEnv.RunningIp
//newEnvironments[envIndex].Licensed = newEnv.Licensed
}
}
} else {
newEnvironments[envIndex].RunningIp = ""
//newEnvironments[envIndex].Licensed = false
newEnvironments[envIndex].DataLake.Enabled = false
}
if len(env.SuborgDistribution) != 0 {
newEnvironments[envIndex].SuborgDistribution = env.SuborgDistribution
}
if newEnvironments[envIndex].Checkin > 0 && timenow-newEnvironments[envIndex].Checkin < 120 {
if len(newEnvironments[envIndex].RunningIp) == 0 {
newEnvironments[envIndex].RunningIp = "IP not available. Check back later."
}
}
}
var newjson []byte
if findEnv != "" && len(newEnvironments) >= 1 {
newjson, err = json.Marshal(newEnvironments[0])
if err != nil {
log.Printf("[DEBUG] Failed unmarshal: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environment"}`)))
return
}
} else {
newjson, err = json.Marshal(newEnvironments)
if err != nil {
log.Printf("[DEBUG] Failed unmarshal: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`)))
return
}
}
//log.Printf("Existing environments: %s", string(newjson))
resp.WriteHeader(200)
resp.Write(newjson)
}
// AutoRepairUserOrgLinks checks all child orgs of the user's active org's parent
// and adds any missing orgs to user.Orgs where user exists in org.Users.
// Should only be called from main region (shuffler). Only runs on cloud.
// Uses cache to avoid running on every request (1 hour TTL).
func AutoRepairUserOrgLinks(ctx context.Context, user *User) int {
// Only run on cloud environment
if project.Environment != "cloud" {
return 0
}
if user == nil || len(user.ActiveOrg.Id) == 0 {
return 0
}
// Skip if we've checked this user recently (1 hour cache)
cacheKey := fmt.Sprintf("orgrepair_%s", user.Id)
if _, err := GetCache(ctx, cacheKey); err == nil {
return 0
}
// Get the active org to find its parent
activeOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] AutoRepairUserOrgLinks: Failed to get active org %s: %s", user.ActiveOrg.Id, err)
return 0
}
// Determine parent org ID
parentOrgId := activeOrg.CreatorOrg
if len(parentOrgId) == 0 {
parentOrgId = activeOrg.Id // Active org is the parent
}
// Get all child orgs
childOrgs, _, err := GetAllChildOrgs(ctx, parentOrgId)
if err != nil {
log.Printf("[WARNING] AutoRepairUserOrgLinks: Failed to get child orgs for %s: %s", parentOrgId, err)
return 0
}
// Build set of orgs user already has
userOrgSet := make(map[string]bool)
for _, orgId := range user.Orgs {
userOrgSet[orgId] = true
}
repairCount := 0
for _, childOrg := range childOrgs {
// Skip if user already has this org
if userOrgSet[childOrg.Id] {
continue
}
// Check if user is in org.Users
for _, orgUser := range childOrg.Users {
if orgUser.Id == user.Id {
log.Printf("[INFO] Auto-repairing user<->org link: user %s (%s) found in org.Users for %s (%s) but missing from user.Orgs",
user.Username, user.Id, childOrg.Name, childOrg.Id)
user.Orgs = append(user.Orgs, childOrg.Id)
userOrgSet[childOrg.Id] = true
repairCount++
break
}
}
}
if repairCount > 0 {
err := SetUser(ctx, user, false)
if err != nil {
log.Printf("[ERROR] Failed saving auto-repaired user.Orgs for %s: %s", user.Username, err)
return 0
}
log.Printf("[INFO] Successfully auto-repaired %d org links for user %s", repairCount, user.Username)
}
// Cache for 1 hour to avoid checking on every request
SetCache(ctx, cacheKey, []byte("1"), 60)
return repairCount
}
func HandleApiAuthentication(resp http.ResponseWriter, request *http.Request) (User, error) {
if request == nil {
return User{}, errors.New("No request given")
}
var err error
apikey := request.Header.Get("Authorization")
org_id := request.Header.Get("Org-Id")
if len(org_id) == 0 {
org_id = request.URL.Query().Get("org_id")
if len(org_id) == 0 {
org_id = request.Header.Get("OrgId")
}
}
user := &User{}
org := &Org{}
ctx := GetContext(request)
if len(org_id) > 0 {
// Get the org
org, err = GetOrg(ctx, org_id)
if err != nil || org.Id != org_id {
//return User{}, errors.New("Invalid org id specified")
log.Printf("[ERROR] Invalid Org-Id specified: %s. Request URL: %#v", org_id, request.URL.String())
org_id = ""
}
}
// Loop headers
if len(apikey) > 0 {
if !strings.HasPrefix(apikey, "Bearer ") {
return User{}, errors.New("No bearer token for authorization header")
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("[WARNING] Invalid format for apikey: %s", apikeyCheck)
return User{}, errors.New("Invalid format for apikey")
}
if len(apikeyCheck[1]) < 36 {
return User{}, errors.New("Apikey must be at least 36 characters long (UUID)")
}
// This is annoying af
newApikey := apikeyCheck[1]
if len(newApikey) > 249 {
newApikey = newApikey[0:248]
}
cache, err := GetCache(ctx, newApikey+org_id)
if err == nil {
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &user)
if err == nil {
//log.Printf("[WARNING] Got user from cache: %s", err)
if len(user.Id) == 0 && len(user.Username) == 0 {
return User{}, errors.New(fmt.Sprintf("Couldn't find user"))
}
user.ApiKey = newApikey
user.SessionLogin = false
// Increment API usage
if user.Username != "scheduler@shuffler.io" {
go IncrementCache(ctx, user.ActiveOrg.Id, "api_usage")
}
return *user, nil
}
} else {
//log.Printf("[WARNING] Error getting authentication cache for %s: %v", newApikey, err)
}
// Make specific check for just service user?
// Get the user based on APIkey here
userdata, err := GetApikey(ctx, apikeyCheck[1])
if err != nil {
// Due to execution auth
if !strings.Contains(request.URL.String(), "authorization=") && !strings.Contains(request.URL.String(), "execution_id=") {
if debug {
log.Printf("[DEBUG] Apikey '%s' doesn't exist. URL: %#v: %s", apikeyCheck[1], request.URL.String(), err)
}
}
return User{}, err
}
if len(userdata.Id) == 0 && len(userdata.Username) == 0 {
//log.Printf("[WARNING] Apikey %s doesn't exist or the user doesn't have an ID/Username", apikey)
return User{}, errors.New("Couldn't find the user")
}
// Caching both bad and good apikeys :)
if len(org_id) > 0 && userdata.ActiveOrg.Id != org_id {
found := false
for _, org := range userdata.Orgs {
if org == org_id {
found = true
break
}
}
if !found {
// VERY specific override to allow ONLY support users in Shuffle to see info for an org to help them out.
if project.Environment == "cloud" && userdata.Verified && userdata.Active && userdata.SupportAccess && strings.HasSuffix(userdata.Username, "@shuffler.io") {
found = true
log.Printf("[AUDIT] User %s (%s) is accessing org %s for support purposes. URL: %#v", userdata.Username, userdata.Id, org_id, request.URL.String())
}
}
if !found {
return User{}, errors.New(fmt.Sprintf("(2) User doesn't have access to org '%s'", org_id))
}
if userdata.ActiveOrg.Id != org_id {
//log.Printf("[AUDIT] Setting user %s (%s) org to %#v FROM %#v for %#v", userdata.Username, userdata.Id, org_id, userdata.ActiveOrg.Id, request.URL.String())
}
userdata.ActiveOrg.Id = org_id
userdata.ActiveOrg.Name = org.Name
userdata.ActiveOrg.Image = org.Image
}
userdata.SessionLogin = false
userdata.ApiKey = newApikey
b, err := json.Marshal(userdata)
if err != nil {
log.Printf("[WARNING] Failed marshalling: %s", err)
return User{}, err
}
err = SetCache(ctx, newApikey+org_id, b, 30)
if err != nil {
log.Printf("[WARNING] Failed setting cache for apikey: %s", err)
}
// Very specific to track schedules in Shuffle
if user.Username != "scheduler@shuffler.io" {
go IncrementCache(ctx, userdata.ActiveOrg.Id, "api_usage")
}
return userdata, nil
}
// One time API keys
//authorizationArr, ok := request.URL.Query()["authorization"]
//if ok {
// //authorization := ""
// //if len(authorizationArr) > 0 {
// // authorization = authorizationArr[0]
// //}
// //_ = authorization
// //log.Printf("[ERROR] WHAT ARE ONE TIME KEYS USED FOR? User input?")
//}
// __session first due to Compatibility issues
c, err := request.Cookie("__session")
if err != nil {
c, err = request.Cookie("session_token")
}
if err == nil {
sessionToken := c.Value
user, err := GetSessionNew(ctx, sessionToken)
if err != nil {
log.Printf("[WARNING] No valid session token for '%s'. Setting cookie to expire. May cause fallback problems.", sessionToken)
if resp != nil {
newCookie := constructSessionDeleteCookie()
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
}
return User{}, err
} else {
// Check if both session tokens are set
// Compatibility issues
//expiration := time.Now().Add(8 * time.Hour)
newCookie := ConstructSessionCookie(sessionToken, c.Expires)
newCookie.MaxAge = c.MaxAge
_, err1 := request.Cookie("session_token")
if err1 != nil {
//log.Printf("[DEBUG] Setting missing session_token for user %s (%s) (1)", user.Username, user.Id)
newCookie.Name = "session_token"
if resp != nil {
http.SetCookie(resp, newCookie)
}
}
_, err2 := request.Cookie("__session")
if err2 != nil {
//log.Printf("[DEBUG] Setting missing __session for user %s (%s) (2)", user.Username, user.Id)
newCookie.Name = "__session"
if resp != nil {
http.SetCookie(resp, newCookie)
}
}
}
if len(user.Id) == 0 && len(user.Username) == 0 {
if resp != nil {
newCookie := constructSessionDeleteCookie()
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
}
return User{}, errors.New(fmt.Sprintf("Couldn't find user"))
}
// This is to be able to overwrite access with available orgs
// Org needs to match one the user already has access to
if len(org_id) > 0 {
found := false
for _, org := range user.Orgs {
if org == org_id {
found = true
break
}
}
if !found {
// VERY specific override to allow ONLY support users in Shuffle to see info for an org to help them out
if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
found = true
}
}
if !found {
return User{}, errors.New(fmt.Sprintf("(1) User doesn't have access to this org (%s)", org_id))
}
if user.ActiveOrg.Id != org_id {
//log.Printf("[AUDIT] Setting user %s (%s) org to %s for %#v", user.Username, user.Id, org_id, request.URL.String())
}
user.ActiveOrg.Id = org_id
user.ActiveOrg.Name = org.Name
user.ActiveOrg.Image = org.Image
}
// We're using the session to find the user anyway, which is NOT user controlled
// This means that this is redundant, but MAY allow users
// to have access past session timeouts
//if user.Session != sessionToken {
// return User{}, errors.New("[WARNING] Wrong session token")
//}
user.SessionLogin = true
// Means session exists, but
return user, nil
}
// Key = apikey
return User{}, errors.New("Missing authentication")
}
func HandleGetUserApps(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
ctx := context.Background()
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in get user apps: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var userId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
userId = location[4]
}
if userId == "me" {
userId = user.Id
}
if user.Id != userId || len(userId) == 0 {
log.Printf("[WARNING] No user ID supplied")
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Supply a valid user ID: /api/v1/users/{userId}/apps"}`))
return
}
userapps, err := GetUserApps(ctx, user.Id)
if err != nil {
log.Printf("[WARNING] Failed getting apps (userapps): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newbody, err := json.Marshal(userapps)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling user apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow apps"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newbody)
}
func GetOpenapi(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in validate swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var id string
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Missing parts of API in request!")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
id = location[4]
}
if len(id) != 32 {
log.Printf("Missing parts of API in request!")
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
parsedApi, openapiErr := GetOpenApiDatastore(ctx, id)
if openapiErr != nil {
log.Printf("[ERROR] Failed getting OpenAPI %s: %s", id, err)
}
app, err := GetApp(ctx, id, user, false)
if err == nil || len(app.ID) > 0 {
log.Printf("[AUDIT] Found app %s (%s) for OpenAPI. Checking for user %s (%s) in org %s (%s) to access", app.Name, id, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
if !app.Public && !app.Sharing && app.Owner != user.Id && user.ActiveOrg.Id != app.ReferenceOrg && !user.SupportAccess {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
} else {
// Try cross region loading?
//if openapiLoad != nil {
//}
}
log.Printf("[INFO] OpenAPI Get length: %d, ID: %s", len(parsedApi.Body), id)
parsedApi.Success = true
data, err := json.Marshal(parsedApi)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling OpenAPI: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling parsed swagger: %s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write(data)
}
func GetActionResult(ctx context.Context, workflowExecution WorkflowExecution, id string) (WorkflowExecution, ActionResult) {
// Get workflow execution to make sure we have the latest
for _, actionResult := range workflowExecution.Results {
if actionResult.Action.ID != id {
continue
}
// ALWAYS relying on cache due to looping subflow issues
if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" {
break
}
if actionResult.Action.AppName == "shuffle-subflow" && project.Environment == "cloud" {
//if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
//log.Printf("[INFO] Skipping due to cache requirement for subflow")
break
}
return workflowExecution, actionResult
}
//log.Printf("[WARNING] No result found for %s - add here too?", id)
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, id)
cache, err := GetCache(ctx, cacheId)
if err == nil {
actionResult := ActionResult{}
cacheData := []byte(cache.([]uint8))
// Just ensuring the data is good
err = json.Unmarshal(cacheData, &actionResult)
if err == nil {
workflowExecution.Results = append(workflowExecution.Results, actionResult)
SetWorkflowExecution(ctx, workflowExecution, false)
return workflowExecution, actionResult
}
}
return workflowExecution, ActionResult{}
}
func GetAction(workflowExecution WorkflowExecution, id, environment string) Action {
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == id {
return action
}
}
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.ID == id {
return Action{
ID: trigger.ID,
AppName: trigger.AppName,
Name: trigger.AppName,
Environment: environment,
Label: trigger.Label,
ExecutionDelay: trigger.ExecutionDelay,
}
log.Printf("[DEBUG] Found trigger to be ran as app (?): %v!", trigger)
}
}
return Action{}
}
func ArrayContainsLower(visited []string, id string) bool {
found := false
for _, item := range visited {
if strings.ToLower(item) == strings.ToLower(id) {
found = true
break
}
}
return found
}
func ArrayContainsInt(visited []int, id int) bool {
found := false
for _, item := range visited {
if item == id {
found = true
break
}
}
return found
}
func ArrayContains(visited []string, id string) bool {
found := false
for _, item := range visited {
if item == id {
found = true
break
}
}
return found
}
func HandleGetWorkflowRunCount(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in getting workflow execution count: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow execution count is not valid"}`))
return
}
// get workflow and verify that it belongs to user
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId, true)
if err != nil {
log.Printf("[WARNING] Failed getting workflow %s while getting runcount: %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
if workflow.OrgId == user.ActiveOrg.Id {
//log.Printf("[AUDIT] User %s is accessing workflow count for '%s' (%s) as %s (get count) in org %s", user.Username, workflow.Name, workflow.ID, user.Role, user.ActiveOrg.Id)
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow run count for %s", user.Username, workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (get workflow run count)", user.Username, workflow.ID)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
}
// FIXME: This is not used yet
// Get the "start_time" and "end_time" query params
// They will be in this format: 2023-12-31T23:00:00.000Z
// By default 30 days back -> 1 day in the future (with 00:00:00 timestamp)
startTimeInt := time.Now().AddDate(0, 0, -30)
endTimeInt := time.Now().AddDate(0, 0, 1)
// Normalize startTimeInt & endTimeInt to be at 00:00:00
startTimeInt = time.Date(startTimeInt.Year(), startTimeInt.Month(), startTimeInt.Day(), 0, 0, 0, 0, startTimeInt.Location())
endTimeInt = time.Date(endTimeInt.Year(), endTimeInt.Month(), endTimeInt.Day(), 0, 0, 0, 0, endTimeInt.Location())
startTime := request.URL.Query().Get("start_time")
endTime := request.URL.Query().Get("end_time")
if len(startTime) != 0 {
// Check if url decode is necessary
if strings.Contains(startTime, "%") {
startTime, err = url.QueryUnescape(startTime)
if err != nil {
log.Printf("[WARNING] Failed url decoding start time '%s': %s", startTime, err)
}
}
// Make starttime 1 year ago
startTimeInt, err = time.Parse(time.RFC3339, startTime)
if err != nil {
log.Printf("[WARNING] Failed parsing start time: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed parsing start time"}`))
return
}
}
if len(endTime) != 0 {
// Check if url decode is necessary
if strings.Contains(endTime, "%") {
endTime, err = url.QueryUnescape(endTime)
if err != nil {
log.Printf("[WARNING] Failed url decoding end time '%s': %s", endTime, err)
}
}
// Make endtime today
endTimeInt, err = time.Parse(time.RFC3339, endTime)
if err != nil {
log.Printf("[WARNING] Failed parsing start time: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed parsing start time"}`))
return
}
}
// Convert to unix timestamp
startTimeNew := startTimeInt.Unix()
endTimeNew := endTimeInt.Unix()
//log.Printf("Start time: %#v, end time: %#v", startTimeNew, endTimeNew)
workflowCount, err := GetWorkflowRunCount(ctx, fileId, startTimeNew, endTimeNew)
if err != nil {
log.Printf("[WARNING] Failed getting workflow count for %s", fileId)
if err.Error() == "Not authorized" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User doesn't belong in this org"}`))
return
}
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "count": %d}`, workflowCount)))
}
func GetWorkflowExecutions(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in getting workflow executions: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow executions is not valid"}`))
return
}
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId, true)
if err != nil {
log.Printf("[WARNING] Failed getting the workflow %s locally (get executions): %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow '%s' (%s) executions as %s (get executions)", user.Username, workflow.Name, workflow.ID, user.Role)
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow execs for %s", user.Username, fileId)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (get workflow execs)", user.Username, workflow.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// Query for the specifci workflowId
//q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(30)
//q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId)
maxAmount := 100
top, topOk := request.URL.Query()["top"]
if topOk && len(top) > 0 {
val, err := strconv.Atoi(top[0])
if err == nil {
maxAmount = val
}
}
if maxAmount > 1000 {
maxAmount = 1000
}
workflowExecutions, err := GetAllWorkflowExecutions(ctx, fileId, maxAmount)
if err != nil {
log.Printf("[WARNING] Failed getting executions for %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(workflowExecutions) == 0 {
resp.WriteHeader(200)
resp.Write([]byte("[]"))
return
}
for index, execution := range workflowExecutions {
newResults := []ActionResult{}
newActions := []Action{}
newTriggers := []Trigger{}
// Results
for _, result := range execution.Results {
newParams := []WorkflowAppActionParameter{}
for _, param := range result.Action.Parameters {
if param.Configuration || strings.Contains(strings.ToLower(param.Name), "user") || strings.Contains(strings.ToLower(param.Name), "key") || strings.Contains(strings.ToLower(param.Name), "pass") {
param.Value = ""
//log.Printf("FOUND CONFIG: %s!!", param.Name)
}
newParams = append(newParams, param)
}
result.Action.Parameters = newParams
newResults = append(newResults, result)
}
// Actions
for _, action := range execution.Workflow.Actions {
newParams := []WorkflowAppActionParameter{}
for _, param := range action.Parameters {
if param.Configuration || strings.Contains(strings.ToLower(param.Name), "user") || strings.Contains(strings.ToLower(param.Name), "key") || strings.Contains(strings.ToLower(param.Name), "pass") {
param.Value = ""
//log.Printf("FOUND CONFIG: %s!!", param.Name)
}
newParams = append(newParams, param)
}
action.Parameters = newParams
newActions = append(newActions, action)
}
for _, trigger := range execution.Workflow.Triggers {
trigger.LargeImage = ""
trigger.SmallImage = ""
newTriggers = append(newTriggers, trigger)
}
workflowExecutions[index].Results = newResults
workflowExecutions[index].Workflow.Actions = newActions
workflowExecutions[index].Workflow.Image = ""
workflowExecutions[index].Workflow.Triggers = newTriggers
// Ensures loading also gives the right, cleaned up data
workflowExecutions[index] = cleanupExecutionNodes(ctx, workflowExecutions[index])
}
newjson, err := json.Marshal(workflowExecutions)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow executions"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func GetExecTimeline(fileId string) []WidgetPointData {
backgroundCtx := context.Background()
// Set cache for it
if len(fileId) == 0 {
return []WidgetPointData{}
}
foundDates := []WidgetPointData{}
cacheId := fmt.Sprintf("exec_timeline_%s", fileId)
cache, err := GetCache(backgroundCtx, cacheId)
if err == nil {
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &foundDates)
if err == nil {
return foundDates
}
}
for i := -30; i < 0; i++ {
startTimeInt := time.Now().AddDate(0, 0, i)
endTimeInt := time.Now().AddDate(0, 0, i+1)
// Normalize startTimeInt & endTimeInt to be at 00:00:00
startTimeInt = time.Date(startTimeInt.Year(), startTimeInt.Month(), startTimeInt.Day(), 0, 0, 0, 0, startTimeInt.Location())
endTimeInt = time.Date(endTimeInt.Year(), endTimeInt.Month(), endTimeInt.Day(), 0, 0, 0, 0, endTimeInt.Location())
startTimeNew := startTimeInt.Unix()
endTimeNew := endTimeInt.Unix()
execCount, err := GetWorkflowRunCount(backgroundCtx, fileId, startTimeNew, endTimeNew)
if err != nil {
log.Printf("[WARNING] Failed getting workflow count for %s: %s", fileId, err)
}
foundDates = append(foundDates, WidgetPointData{
Key: startTimeInt.Format("Jan 2"),
Data: int64(execCount),
MetaData: WidgetMeta{
Color: "grey",
},
})
}
// Set cache for it
if project.CacheDb {
cacheData, err := json.Marshal(foundDates)
if err != nil {
log.Printf("[WARNING] Failed marshalling exec timeline data: %s", err)
} else {
err = SetCache(backgroundCtx, cacheId, cacheData, 400) // 1 week
}
}
return foundDates
}
func GetWorkflowExecutionsV2(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in getting workflow executions: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow executions is not valid"}`))
return
}
ctx := GetContext(request)
checkExecOrg := false
workflow, err := GetWorkflow(ctx, fileId, true)
if err != nil {
log.Printf("[WARNING] Failed getting the workflow %s locally (get executions v2): %s", fileId, err)
checkExecOrg = true
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false}`))
//return
}
if user.Id != workflow.Owner || len(user.Id) == 0 {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s (%s) is accessing workflow '%s' (%s) executions V2 as %s (get executions)", user.Username, user.Id, workflow.Name, workflow.ID, user.Role)
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow execs (V2) for %s", user.Username, fileId)
checkExecOrg = false
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (get workflow execs)", user.Username, workflow.ID)
checkExecOrg = true
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false}`))
//return
}
}
// Query for the specifci workflowId
maxAmount := 50
top, topOk := request.URL.Query()["top"]
if topOk && len(top) > 0 {
val, err := strconv.Atoi(top[0])
if err == nil {
maxAmount = val
}
}
if maxAmount > 1000 {
maxAmount = 1000
}
// Add timeout of 6 seconds to the ctx
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
cursor := ""
cursorList, cursorOk := request.URL.Query()["cursor"]
if cursorOk && len(cursorList) > 0 {
cursor = cursorList[0]
}
if maxAmount != 50 {
log.Printf("[DEBUG] Getting %d executions for workflow %s (V2). Org %s (%s).", maxAmount, fileId, user.ActiveOrg.Name, user.ActiveOrg.Id)
}
workflowExecutions, newCursor, err := GetAllWorkflowExecutionsV2(ctx, fileId, maxAmount, cursor)
if err != nil {
log.Printf("[WARNING] Failed getting executions for %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if checkExecOrg {
if len(workflowExecutions) != 1 {
log.Printf("[WARNING] Wrong user (%s) for workflow %s (get workflow execs) - not 1", user.Username, workflow.ID)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
if workflowExecutions[0].OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] Wrong user (%s) for workflow %s (get workflow execs) - execution orgid", user.Username, workflow.ID)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
}
if len(workflowExecutions) != maxAmount {
//log.Printf("[DEBUG] Got %d executions for workflow %s (V2). Org %s (%s).", len(workflowExecutions), fileId, user.ActiveOrg.Name, user.ActiveOrg.Id)
}
if len(workflowExecutions) == 0 {
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "executions": [], "cursor": ""}`))
return
}
for index, execution := range workflowExecutions {
if project.Environment != "cloud" && execution.Status != "FINISHED" && execution.Status != "ABORTED" {
execution, _ = Fixexecution(ctx, execution)
}
newResults := []ActionResult{}
newActions := []Action{}
newTriggers := []Trigger{}
// Results
for _, result := range execution.Results {
newParams := []WorkflowAppActionParameter{}
for _, param := range result.Action.Parameters {
if param.Configuration || strings.Contains(strings.ToLower(param.Name), "user") || strings.Contains(strings.ToLower(param.Name), "key") || strings.Contains(strings.ToLower(param.Name), "pass") {
param.Value = ""
//log.Printf("FOUND CONFIG: %s!!", param.Name)
}
newParams = append(newParams, param)
}
result.Action.Parameters = newParams
newResults = append(newResults, result)
}
// Actions
for _, action := range execution.Workflow.Actions {
newParams := []WorkflowAppActionParameter{}
for _, param := range action.Parameters {
if param.Configuration || strings.Contains(strings.ToLower(param.Name), "user") || strings.Contains(strings.ToLower(param.Name), "key") || strings.Contains(strings.ToLower(param.Name), "pass") {
param.Value = ""
//log.Printf("FOUND CONFIG: %s!!", param.Name)
}
newParams = append(newParams, param)
}
action.Parameters = newParams
newActions = append(newActions, action)
}
for _, trigger := range execution.Workflow.Triggers {
trigger.LargeImage = ""
trigger.SmallImage = ""
newTriggers = append(newTriggers, trigger)
}
workflowExecutions[index].Results = newResults
workflowExecutions[index].Workflow.Actions = newActions
workflowExecutions[index].Workflow.Image = ""
workflowExecutions[index].Workflow.Triggers = newTriggers
if workflowExecutions[index].Status != "EXECUTION" && workflowExecutions[index].Workflow.Validation.Valid == false && len(workflowExecutions[index].Workflow.Validation.Errors) == 0 && len(workflowExecutions[index].Workflow.Validation.SubflowApps) == 0 {
validation, err := GetExecutionValidation(ctx, workflowExecutions[index].ExecutionId)
if err == nil {
workflowExecutions[index].Workflow.Validation = validation
}
}
workflowExecutions[index] = cleanupExecutionNodes(ctx, workflowExecutions[index])
}
newReturn := ExecutionReturn{
Success: true,
Id: fileId,
Cursor: newCursor,
Executions: workflowExecutions,
Timeline: GetExecTimeline(fileId),
}
newjson, err := json.Marshal(newReturn)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow executions"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func GetWorkflows(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in getworkflows: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
var workflows []Workflow
maxAmount := 250
top, topOk := request.URL.Query()["top"]
if topOk && len(top) > 0 {
val, err := strconv.Atoi(top[0])
if err == nil {
maxAmount = val
}
}
cursor := ""
cursorList, cursorOk := request.URL.Query()["cursor"]
if cursorOk && len(cursorList) > 0 {
cursor = cursorList[0]
}
skipTruncate := false
truncate, truncateOk := request.URL.Query()["truncate"]
if truncateOk && len(truncate) > 0 && truncate[0] == "false" {
skipTruncate = true
}
workflows, err = GetAllWorkflowsByQuery(ctx, user, maxAmount, cursor)
if err != nil {
log.Printf("[WARNING] Failed getting workflows for user %s (0): %s", user.Username, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if len(workflows) == 0 {
log.Printf("[INFO] No workflows found for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte("[]"))
return
}
if skipTruncate == true {
newjson, err := json.Marshal(workflows)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling workflows: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking untruncated workflows"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
return
}
usecaseIds := []string{}
parentWorkflows := []Workflow{}
for _, workflow := range workflows {
if workflow.OrgId != user.ActiveOrg.Id {
if !ArrayContains(workflow.SuborgDistribution, user.ActiveOrg.Id) {
continue
}
}
if workflow.Hidden {
continue
}
if project.Environment == "cloud" && workflow.ExecutionEnvironment == "onprem" {
continue
}
newActions := []Action{}
for _, action := range workflow.Actions {
// Removed because of exports. These are needed there.
//action.LargeImage = ""
//action.SmallImage = ""
action.ReferenceUrl = ""
newActions = append(newActions, action)
}
//workflow.Actions = newActions
// Skipping these as they're related to onprem workflows in cloud (orborus)
usecaseIds = append(usecaseIds, workflow.UsecaseIds...)
parentWorkflows = append(parentWorkflows, workflow)
}
//log.Printf("[DEBUG] Env: %s, workflows: %d", project.Environment, len(parentWorkflows))
if project.Environment == "cloud" && len(parentWorkflows) > 40 {
//if debug {
// log.Printf("[DEBUG] Removed workflow actions & images for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
//}
// Check for "subflow" query
isSubflow := false
if subflow, subflowOk := request.URL.Query()["subflow"]; subflowOk && len(subflow) > 0 {
if subflow[0] == "true" {
isSubflow = true
}
}
for workflowIndex, _ := range parentWorkflows {
if workflowIndex < 4 {
continue
}
if !isSubflow {
//parentWorkflows[workflowIndex].Actions = []Action{}
parentWorkflows[workflowIndex].Image = ""
}
parentWorkflows[workflowIndex].Branches = []Branch{}
parentWorkflows[workflowIndex].VisualBranches = []Branch{}
parentWorkflows[workflowIndex].Description = ""
parentWorkflows[workflowIndex].Blogpost = ""
if len(parentWorkflows[workflowIndex].Org) > 0 {
for orgIndex, _ := range parentWorkflows[workflowIndex].Org {
parentWorkflows[workflowIndex].Org[orgIndex].Image = ""
}
}
parentWorkflows[workflowIndex].ExecutingOrg.Image = ""
}
// Add header that this is a limited response
resp.Header().Set("X-Shuffle-Truncated", "true")
} else {
//log.Printf("[DEBUG] Loading workflows without truncating for user %s (%s) in org %s (%s)", user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
}
// Get the org as well to manage priorities
// Only happens on first load, so it's like once per session~
if len(usecaseIds) > 0 {
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org %s for user %s during workflow load: %s", user.ActiveOrg.Id, user.Username, err)
} else {
for prioIndex, priority := range org.Priorities {
if priority.Type != "usecase" || priority.Active != true {
continue
}
for _, usecaseId := range usecaseIds {
if strings.Contains(strings.ToLower(priority.Name), strings.ToLower(usecaseId)) {
//log.Printf("\n\n[DEBUG] Found usecase %s in priority %s\n\n", usecaseId, priority.Name)
org.Priorities[prioIndex].Active = false
SetOrg(ctx, *org, org.Id)
break
}
}
}
}
}
// Fix parent/child workflow loading to only load EITHER parent OR child
removeIds := []string{}
newParsedWorkflows := []Workflow{}
for _, workflow := range parentWorkflows {
if len(workflow.ChildWorkflowIds) > 0 {
found := false
for _, childId := range workflow.ChildWorkflowIds {
for _, checkWorkflow := range parentWorkflows {
if checkWorkflow.ID == childId {
found = true
break
}
}
if found {
break
}
}
if found {
continue
}
}
if len(workflow.ParentWorkflowId) > 0 {
removeIds = append(removeIds, workflow.ParentWorkflowId)
}
newParsedWorkflows = append(newParsedWorkflows, workflow)
}
// Bleh
if len(removeIds) > 0 {
anotherNewOne := []Workflow{}
for _, newParsed := range newParsedWorkflows {
if ArrayContains(removeIds, newParsed.ID) {
continue
}
anotherNewOne = append(anotherNewOne, newParsed)
}
newParsedWorkflows = anotherNewOne
}
parentWorkflows = newParsedWorkflows
//log.Printf("[INFO] Returning %d workflows", len(parentWorkflows))
newjson, err := json.Marshal(parentWorkflows)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflows"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func SetAuthenticationConfig(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in get all apps: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin during auth edit config")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 5 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[5]
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type configAuth struct {
Id string `json:"id"`
Action string `json:"action"`
SelectedSuborg []string `json:"selected_suborgs"`
}
var config configAuth
err = json.Unmarshal(body, &config)
if err != nil {
log.Printf("Failed unmarshaling (appauth): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if config.Id != fileId {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Bad ID match"}`))
return
}
ctx := GetContext(request)
auth, err := GetWorkflowAppAuthDatastore(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Authget error: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": ":("}`))
return
}
if auth.OrgId != user.ActiveOrg.Id {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "User can't edit this org"}`))
return
}
if config.Action == "assign_everywhere" {
err := AssignAuthEverywhere(ctx, auth, user)
if err != nil {
log.Printf("[ERROR] Failed assigning auth everywhere: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting workflows to update"}`))
} else {
log.Printf("[INFO] Assigned auth everywhere")
}
} else if config.Action == "suborg_distribute" {
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", auth.OrgId, err)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org"}`))
return
}
// Check if org doesn't have a creator org
if len(org.CreatorOrg) != 0 {
log.Printf("[INFO] Org %s has creator org %s, can't distribute", org.Id, org.CreatorOrg)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Can't distribute auth for suborgs"}`))
return
}
if len(config.SelectedSuborg) == 0 {
auth.SuborgDistribution = []string{}
auth.SuborgDistributed = false
} else {
auth.SuborgDistribution = config.SelectedSuborg
auth.SuborgDistributed = false
}
err = SetWorkflowAppAuthDatastore(ctx, *auth, auth.Id)
if err != nil {
log.Printf("[ERROR] Failed setting auth for org %s (%s): %s", org.Name, org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed updating auth. Please try again."}`))
return
}
for _, childOrg := range org.ChildOrgs {
nameKey := "workflowappauth"
cacheKey := fmt.Sprintf("%s_%s", nameKey, childOrg.Id)
DeleteCache(ctx, cacheKey)
}
} else {
log.Printf("[WARNING] Unknown auth change action %s", config.Action)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
//var config configAuth
//log.Printf("Should set %s
}
func findAuthForApp(appId string, allAuths []AppAuthenticationStorage) string {
var activeAuth *AppAuthenticationStorage
var firstAuth *AppAuthenticationStorage
for i, auth := range allAuths {
if auth.App.ID != appId {
continue
}
if firstAuth == nil {
firstAuth = &allAuths[i]
}
if auth.Active {
activeAuth = &allAuths[i]
break
}
}
if activeAuth != nil {
return activeAuth.Id
}
if firstAuth != nil {
return firstAuth.Id
}
return ""
}
func SetAuthenticationConfigBatch(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in batch auth config: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin during batch auth edit config")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[ERROR] Error with body read in batch auth config: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type configItem struct {
Id string `json:"id"`
SelectedSuborg []string `json:"selected_suborgs"`
}
type batchConfigAuth struct {
Action string `json:"action"`
Type string `json:"type"`
Configs []configItem `json:"configs"`
}
var config batchConfigAuth
err = json.Unmarshal(body, &config)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling batch auth config: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid request body"}`))
return
}
if config.Type != "auth_id" && config.Type != "app_id" {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Type must be either 'auth_id' or 'app_id'"}`))
return
}
if len(config.Configs) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No configs provided"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org"}`))
return
}
if len(org.CreatorOrg) != 0 {
log.Printf("[INFO] Org %s has creator org %s, can't distribute", org.Id, org.CreatorOrg)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Can't distribute auth for suborgs"}`))
return
}
type resultItem struct {
Id string `json:"id"`
AuthId string `json:"auth_id,omitempty"`
Success bool `json:"success"`
Reason string `json:"reason,omitempty"`
}
type batchResponse struct {
Success bool `json:"success"`
Results []resultItem `json:"results"`
}
results := []resultItem{}
authConfigsToProcess := []configItem{}
appIdToAuthIdMap := make(map[string]string)
if config.Type == "app_id" {
allAuths, err := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting all auths for org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting auths"}`))
return
}
for _, appConfig := range config.Configs {
authId := findAuthForApp(appConfig.Id, allAuths)
if authId == "" {
result := resultItem{
Id: appConfig.Id,
Success: false,
Reason: "No auth found for app",
}
results = append(results, result)
continue
}
appIdToAuthIdMap[authId] = appConfig.Id
authConfigsToProcess = append(authConfigsToProcess, configItem{
Id: authId,
SelectedSuborg: appConfig.SelectedSuborg,
})
}
} else {
authConfigsToProcess = config.Configs
}
if config.Action == "suborg_distribute" {
for _, authConfig := range authConfigsToProcess {
result := resultItem{
Id: authConfig.Id,
Success: false,
}
if appId, exists := appIdToAuthIdMap[authConfig.Id]; exists {
result.Id = appId
result.AuthId = authConfig.Id
}
auth, err := GetWorkflowAppAuthDatastore(ctx, authConfig.Id)
if err != nil {
log.Printf("[WARNING] Failed getting auth %s: %s", authConfig.Id, err)
result.Reason = "Auth doesn't exist"
results = append(results, result)
continue
}
if auth.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s can't edit auth %s from org %s", user.Id, authConfig.Id, auth.OrgId)
result.Reason = "User can't edit this org"
results = append(results, result)
continue
}
if len(authConfig.SelectedSuborg) == 0 {
auth.SuborgDistribution = []string{}
auth.SuborgDistributed = false
} else {
auth.SuborgDistribution = authConfig.SelectedSuborg
auth.SuborgDistributed = false
}
err = SetWorkflowAppAuthDatastore(ctx, *auth, auth.Id)
if err != nil {
log.Printf("[ERROR] Failed setting auth %s for org %s: %s", authConfig.Id, org.Id, err)
result.Reason = "Failed updating auth"
results = append(results, result)
continue
}
result.Success = true
results = append(results, result)
}
for _, childOrg := range org.ChildOrgs {
nameKey := "workflowappauth"
cacheKey := fmt.Sprintf("%s_%s", nameKey, childOrg.Id)
DeleteCache(ctx, cacheKey)
}
} else {
log.Printf("[WARNING] Unknown batch auth change action %s", config.Action)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Invalid action provided"}`))
return
}
responseData := batchResponse{
Success: true,
Results: results,
}
responseJson, err := json.Marshal(responseData)
if err != nil {
log.Printf("[ERROR] Failed marshaling batch response: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed creating response"}`))
return
}
resp.WriteHeader(200)
resp.Write(responseJson)
}
func DistributeWorkflowsBatch(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in batch workflow distribution: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin during batch workflow distribution")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[ERROR] Error with body read in batch workflow distribution: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type workflowConfigItem struct {
WorkflowId string `json:"workflow_id"`
SuborgIds []string `json:"suborg_ids"`
}
type batchWorkflowDistribution struct {
Workflows []workflowConfigItem `json:"workflows"`
}
var config batchWorkflowDistribution
err = json.Unmarshal(body, &config)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling batch workflow distribution: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid request body"}`))
return
}
if len(config.Workflows) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No workflows provided"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org"}`))
return
}
if len(org.CreatorOrg) != 0 {
log.Printf("[INFO] Org %s has creator org %s, can't distribute workflows", org.Id, org.CreatorOrg)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Can't distribute workflows from suborg"}`))
return
}
validSuborgIds := make(map[string]bool)
for _, childOrg := range org.ChildOrgs {
validSuborgIds[childOrg.Id] = true
}
type distributionResult struct {
SuborgId string `json:"suborg_id"`
ChildWorkflowId string `json:"child_workflow_id,omitempty"`
Success bool `json:"success"`
Reason string `json:"reason,omitempty"`
}
type workflowResult struct {
WorkflowId string `json:"workflow_id"`
Success bool `json:"success"`
Distributions []distributionResult `json:"distributions"`
}
type batchResponse struct {
Success bool `json:"success"`
Results []workflowResult `json:"results"`
}
results := []workflowResult{}
for _, workflowConfig := range config.Workflows {
result := workflowResult{
WorkflowId: workflowConfig.WorkflowId,
Success: false,
Distributions: []distributionResult{},
}
workflow, err := GetWorkflow(ctx, workflowConfig.WorkflowId)
if err != nil {
log.Printf("[WARNING] Failed getting workflow %s: %s", workflowConfig.WorkflowId, err)
result.Distributions = append(result.Distributions, distributionResult{
Success: false,
Reason: "Workflow doesn't exist",
})
results = append(results, result)
continue
}
if workflow.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s can't distribute workflow %s from org %s", user.Id, workflowConfig.WorkflowId, workflow.OrgId)
result.Distributions = append(result.Distributions, distributionResult{
Success: false,
Reason: "Workflow doesn't belong to your org",
})
results = append(results, result)
continue
}
if len(workflow.ParentWorkflowId) > 0 {
log.Printf("[WARNING] Workflow %s is a child workflow, can't distribute", workflowConfig.WorkflowId)
result.Distributions = append(result.Distributions, distributionResult{
Success: false,
Reason: "Can't distribute child workflows",
})
results = append(results, result)
continue
}
allSuccess := true
for _, suborgId := range workflowConfig.SuborgIds {
distResult := distributionResult{
SuborgId: suborgId,
Success: false,
}
if !validSuborgIds[suborgId] {
log.Printf("[WARNING] Suborg %s is not a child of org %s", suborgId, org.Id)
distResult.Reason = "Suborg doesn't belong to your org"
result.Distributions = append(result.Distributions, distResult)
allSuccess = false
continue
}
childWorkflow, err := GenerateWorkflowFromParent(ctx, *workflow, org.Id, suborgId)
if err != nil {
log.Printf("[ERROR] Failed generating child workflow for %s in suborg %s: %s", workflowConfig.WorkflowId, suborgId, err)
distResult.Reason = "Failed creating child workflow"
result.Distributions = append(result.Distributions, distResult)
allSuccess = false
continue
}
distResult.Success = true
distResult.ChildWorkflowId = childWorkflow.ID
result.Distributions = append(result.Distributions, distResult)
}
result.Success = allSuccess
results = append(results, result)
}
for _, childOrg := range org.ChildOrgs {
cacheKey := fmt.Sprintf("%s_workflows", childOrg.Id)
DeleteCache(ctx, cacheKey)
}
responseData := batchResponse{
Success: true,
Results: results,
}
responseJson, err := json.Marshal(responseData)
if err != nil {
log.Printf("[ERROR] Failed marshaling batch workflow distribution response: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed creating response"}`))
return
}
resp.WriteHeader(200)
resp.Write(responseJson)
}
func HandleGetTriggers(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get schedules: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success":false}`))
return
}
ctx := GetContext(request)
workflowsChan := make(chan []Workflow)
schedulesChan := make(chan []ScheduleOld)
hooksChan := make(chan []Hook)
pipelinesChan := make(chan []PipelineInfo)
errChan := make(chan error)
wg := sync.WaitGroup{}
wg.Add(4)
go func() {
workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "")
if err != nil {
wg.Done()
errChan <- err
return
}
wg.Done()
workflowsChan <- workflows
}()
go func() {
schedules, err := GetAllSchedules(ctx, user.ActiveOrg.Id)
if err != nil {
wg.Done()
errChan <- err
return
}
wg.Done()
schedulesChan <- schedules
}()
go func() {
hooks, err := GetHooks(ctx, user.ActiveOrg.Id)
if err != nil {
wg.Done()
errChan <- err
return
}
wg.Done()
hooksChan <- hooks
}()
go func() {
// List it out from Environments and track them from each node directly as this is the most up to date config
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
wg.Done()
errChan <- err
return
}
pipelines := []PipelineInfo{}
for _, env := range environments {
if env.Archived {
continue
}
cacheKey := fmt.Sprintf("queueconfig-%s-%s", env.Name, env.OrgId)
cache, err := GetCache(ctx, cacheKey)
if err == nil {
newEnv := OrborusStats{}
err = json.Unmarshal(cache.([]uint8), &newEnv)
if err == nil {
for _, pipeline := range newEnv.DataLake.Pipelines {
pipeline.Environment = env.Name
pipelines = append(pipelines, pipeline)
}
}
}
}
//log.Printf("[DEBUG] Found %d pipelines in %d environments in org %s (%s)", len(pipelines), len(environments), user.ActiveOrg.Name, user.ActiveOrg.Id)
/*
pipelines, err := GetPipelines(ctx, user.ActiveOrg.Id)
if err != nil {
wg.Done()
errChan <- err
return
}
*/
wg.Done()
pipelinesChan <- pipelines
}()
wg.Wait()
// Checks if we got any errors without blocking the entire process
select {
case err := <-errChan:
if err != nil {
log.Printf("[ERROR] Failed to fetch data: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success":false}`))
return
}
default:
//log.Println("[INFO] No errors received within Go routines, proceeding with further logic")
}
hooks := <-hooksChan
schedules := <-schedulesChan
workflows := <-workflowsChan
pipelines := <-pipelinesChan
hookMap := map[string]Hook{}
scheduleMap := map[string]ScheduleOld{}
pipelineMap := map[string]PipelineInfo{}
for _, hook := range hooks {
hookMap[hook.Id] = hook
}
for _, schedule := range schedules {
scheduleMap[schedule.Id] = schedule
}
for _, pipeline := range pipelines {
pipelineMap[pipeline.ID] = pipeline
}
allHooks := []Hook{}
allSchedules := []ScheduleOld{}
// Now loop through the workflow triggers to see if anything is not in sync
for _, workflow := range workflows {
for _, trigger := range workflow.Triggers {
/*
if trigger.Status == "uninitialized" {
continue
}
*/
switch trigger.TriggerType {
case "WEBHOOK":
{
hook := Hook{}
storedHook, exist := hookMap[trigger.ID]
if !exist {
auth := ""
version := ""
customBody := ""
startNode := ""
hook.Id = trigger.ID
hook.Environment = trigger.Environment
hook.Workflows = []string{workflow.ID}
hook.Owner = workflow.Owner
hook.OrgId = workflow.OrgId
hookInfo := Info{}
for _, param := range trigger.Parameters {
if param.Name == "url" {
hookInfo.Url = param.Value
hookInfo.Name = trigger.Label
} else if param.Name == "auth_headers" {
auth = param.Value
} else if param.Name == "await_response" {
version = param.Value
} else if param.Name == "custom_response_body" {
customBody = param.Value
}
}
hook.Info = hookInfo
// searching for start node
if len(workflow.Branches) != 0 {
for _, branch := range workflow.Branches {
if branch.SourceID == trigger.ID {
startNode = branch.DestinationID
}
}
}
if startNode == "" {
startNode = workflow.Start
}
hook.Start = startNode
hook.Status = "stopped"
hook.Running = false
hook.Auth = auth
hook.Version = version
hook.CustomResponse = customBody
allHooks = append(allHooks, hook)
} else {
hookValue := storedHook
//hookValue.Status = "running"
for _, param := range trigger.Parameters {
if param.Name == "url" {
hookValue.Info.Url = param.Value
hookValue.Info.Name = trigger.Label
}
}
allHooks = append(allHooks, hookValue)
}
}
case "SCHEDULE":
{
schedule := ScheduleOld{}
storedschedule, exist := scheduleMap[trigger.ID]
if !exist {
startNode := ""
schedule.Id = trigger.ID
schedule.WorkflowId = workflow.ID
schedule.Environment = trigger.Environment
schedule.Org = workflow.OrgId
schedule.Name = trigger.Label
for _, param := range trigger.Parameters {
if param.Name == "cron" {
schedule.Frequency = param.Value
} else if param.Name == "execution_argument" {
schedule.Argument = param.Value
}
}
for _, branch := range workflow.Branches {
if branch.SourceID == schedule.Id {
startNode = branch.DestinationID
}
}
if startNode == "" {
startNode = workflow.Start
}
schedule.StartNode = startNode
Wrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "schedule", "execution_argument": "%s"}`, startNode, schedule.Argument)
schedule.WrappedArgument = Wrapper
schedule.Status = "stopped"
allSchedules = append(allSchedules, schedule)
} else {
if project.Environment != "cloud" && storedschedule.Status == "" {
storedschedule.Status = "running"
}
scheduleValue := storedschedule
scheduleValue.Name = trigger.Label
//scheduleValue.Status = "running"
allSchedules = append(allSchedules, scheduleValue)
}
}
case "PIPELINE":
{
// Handled otherwise.
/*
storedPipeline, exist := pipelineMap[trigger.ID]
if exist && storedPipeline.Status != "uninitialized" {
startNode := ""
storedPipeline.WorkflowId = workflow.ID
if len(workflow.Branches) != 0 {
for _, branch := range workflow.Branches {
if branch.SourceID == trigger.ID {
startNode = branch.DestinationID
}
}
}
if startNode == "" {
startNode = workflow.Start
}
storedPipeline.StartNode = startNode
allPipelines = append(allPipelines, storedPipeline)
}
*/
}
}
}
}
if project.Environment == "cloud" {
var wg sync.WaitGroup
scheduleMutex := sync.Mutex{}
for index, schedule := range allSchedules {
wg.Add(1)
go func(index int, schedule ScheduleOld) {
defer wg.Done()
// Check if the schedule exist in the gcp
GcpSchedule, err := GetGcpSchedule(ctx, schedule.Id)
// Use mutex to safely update the schedule status
scheduleMutex.Lock()
if err != nil {
allSchedules[index].Status = "stopped"
} else {
allSchedules[index].Status = GcpSchedule.Status
}
scheduleMutex.Unlock()
}(index, schedule)
}
wg.Wait()
}
sort.SliceStable(allHooks, func(i, j int) bool {
return allHooks[i].Info.Name < allHooks[j].Info.Name
})
sort.SliceStable(allSchedules, func(i, j int) bool {
return allSchedules[i].Name < allSchedules[j].Name
})
sort.SliceStable(pipelines, func(i, j int) bool {
return pipelines[i].Name < pipelines[j].Name
})
allTriggersWrapper := AllTriggersWrapper{}
allTriggersWrapper.WebHooks = allHooks
allTriggersWrapper.Schedules = allSchedules
allTriggersWrapper.Pipelines = pipelines
newjson, err := json.Marshal(allTriggersWrapper)
if err != nil {
log.Printf("Failed unmarshal: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed unpacking environments"}`))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func GetGcpSchedule(ctx context.Context, id string) (*ScheduleOld, error) {
if project.Environment != "cloud" {
return &ScheduleOld{}, nil
}
// Check if we have the schedule in cache
cacheData, err := GetCache(ctx, fmt.Sprintf("schedule-%s", id))
if err == nil {
data, ok := cacheData.([]byte)
if !ok {
log.Printf("[ERROR] Cache data for %s is not of type []byte", id)
} else {
schedule := &ScheduleOld{}
err = json.Unmarshal(data, schedule)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal schedule cache for %s: %s", id, err)
} else {
return schedule, nil
}
}
}
schedule := &ScheduleOld{}
c, err := scheduler.NewCloudSchedulerClient(ctx)
if err != nil {
log.Printf("[ERROR] Client error: %s", err)
return schedule, err
}
location := "europe-west2"
if len(os.Getenv("SHUFFLE_GCE_LOCATION")) > 0 {
location = os.Getenv("SHUFFLE_GCE_LOCATION")
}
if len(os.Getenv("SHUFFLE_GCE_LOCATION")) == 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
location = os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")
}
req := &schedulerpb.GetJobRequest{
Name: fmt.Sprintf("projects/%s/locations/%s/jobs/schedule_%s", gceProject, location, id),
}
resp, err := c.GetJob(ctx, req)
if err != nil {
if !strings.Contains(err.Error(), "NotFound") {
log.Printf("[ERROR] Failed getting schedule %s: %s", id, err)
}
return schedule, err
}
schedule.Id = id
schedule.Name = resp.Name
if resp.State == schedulerpb.Job_ENABLED {
schedule.Status = "running"
} else {
schedule.Status = "stopped"
}
// Set cache for 5 minutes just to make it fast
scheduleJSON, err := json.Marshal(schedule)
if err != nil {
log.Printf("[ERROR] Failed to marshal schedule for cache: %s", err)
return schedule, err
}
err = SetCache(ctx, fmt.Sprintf("schedule-%s", id), scheduleJSON, 300)
if err != nil {
log.Printf("[ERROR] Failed setting cache for schedule %s: %s", id, err)
}
return schedule, nil
}
func HandleGetSchedules(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get schedules: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Admin required"}`))
return
}
ctx := GetContext(request)
schedules, err := GetAllSchedules(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting schedules: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Couldn't get schedules"}`))
return
}
newjson, err := json.Marshal(schedules)
if err != nil {
log.Printf("Failed unmarshal: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`)))
return
}
//log.Printf("Existing environments: %s", string(newjson))
resp.WriteHeader(200)
resp.Write(newjson)
}
func HandleGetHooks(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get hooks: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Admin required"}`))
return
}
ctx := GetContext(request)
hooks, err := GetHooks(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting hooks: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Couldn't get hooks"}`))
return
}
newjson, err := json.Marshal(hooks)
if err != nil {
log.Printf("Failed unmarshal: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func HandleUpdateUser(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Update User request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
userInfo, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in update user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body in update user: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Required field: user_id"}`)))
return
}
// NEVER allow the user to set all the data themselves
type newUserStruct struct {
UserId string `json:"user_id"`
Tutorial string `json:"tutorial" datastore:"tutorial"`
Firstname string `json:"firstname"`
Lastname string `json:"lastname"`
Role string `json:"role"`
Username string `json:"username"`
CompanyRole string `json:"company_role"`
Suborgs []string `json:"suborgs"`
CreatorDescription string `json:"creator_description"`
CreatorUrl string `json:"creator_url"`
CreatorLocation string `json:"creator_location"`
CreatorSkills string `json:"creator_skills"`
CreatorWorkStatus string `json:"creator_work_status"`
CreatorSocial string `json:"creator_social"`
SpecializedApps []MinimizedApps `json:"specialized_apps"`
Theme string `json:"theme"`
}
ctx := GetContext(request)
var t newUserStruct
err = json.Unmarshal(body, &t)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling userId: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unmarshaling. Required field: user_id"}`)))
return
}
// Should this role reflect the users' org access?
// When you change org -> change user role
if userInfo.Role != "admin" && userInfo.Id != t.UserId {
log.Printf("[WARNING] User %s tried to update user %s. Role: %s", userInfo.Username, t.UserId, userInfo.Role)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to be admin to change other users"}`)))
return
}
foundUser, err := GetUser(ctx, t.UserId)
if err != nil {
log.Printf("[WARNING] Can't find user %s (update user): %s", t.UserId, err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
defaultRole := foundUser.Role
orgFound := false
for _, item := range foundUser.Orgs {
if item == userInfo.ActiveOrg.Id {
orgFound = true
break
}
}
if !orgFound {
isSelf := false
if project.Environment == "cloud" && len(foundUser.Id) == 32 {
isSelf = CheckCreatorSelfPermission(ctx, userInfo, *foundUser, &AlgoliaSearchCreator{ObjectID: foundUser.Id, IsOrg: true})
}
if (!isSelf || len(foundUser.Id) != 32) && !userInfo.SupportAccess {
log.Printf("[AUDIT] User %s (%s) is admin, but can't edit users outside their own org (%s).", userInfo.Username, userInfo.Id, foundUser.Id)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You don't have access to modify this user. Contact support@shuffler.io if you think this is wrong."}`)))
return
}
}
orgUpdater := true
if len(t.Role) > 0 && (t.Role != "admin" && t.Role != "user" && t.Role != "org-reader") {
log.Printf("[WARNING] %s tried and failed to update user %s", userInfo.Username, t.UserId)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can only change to roles user, admin and org-reader"}`)))
return
} else {
// Same user - can't edit yourself?
if len(t.Role) > 0 && (userInfo.Id == t.UserId || userInfo.Username == t.UserId) {
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't update the role of your own user"}`)))
return
}
oldRole := foundUser.Role
if len(t.Role) > 0 {
orgUpdater = false
// Realtime update if the user is in the same org
if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id {
foundUser.Role = t.Role
foundUser.Roles = []string{t.Role}
foundUser.ActiveOrg.Role = t.Role
err = SetUser(ctx, foundUser, false)
if err != nil {
log.Printf("[ERROR] Failed setting user when changing role to %s for %s (%s): %s", t.Role, foundUser.Username, foundUser.Id, err)
}
}
// Getting the specific org and just updating the user in that one
foundOrg, err := GetOrg(ctx, userInfo.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed to get org in edit role to %s for %s (%s): %s", t.Role, foundUser.Username, foundUser.Id, err)
} else {
users := []User{}
for _, user := range foundOrg.Users {
if user.Id == foundUser.Id {
user.Role = t.Role
user.Roles = []string{t.Role}
// Avoids role bypassing by setting the role in the active org
if user.ActiveOrg.Id == foundOrg.Id {
user.ActiveOrg.Role = t.Role
}
}
users = append(users, user)
}
foundOrg.Users = users
err = SetOrg(ctx, *foundOrg, foundOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed setting org when changing role to %s for %s (%s): %s", t.Role, foundUser.Username, foundUser.Id, err)
}
}
}
if len(t.Role) > 0 {
log.Printf("[INFO] Updated user '%s' from '%s' to '%s' in org %s.", foundUser.Username, oldRole, t.Role, userInfo.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
return
}
}
if len(t.Username) > 0 && project.Environment != "cloud" {
users, err := FindUser(ctx, strings.ToLower(strings.TrimSpace(t.Username)))
if err != nil && len(users) == 0 {
log.Printf("[WARNING] Failed getting user %s: %s", t.Username, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
found := false
for _, item := range users {
if item.Username == t.Username {
found = true
break
}
}
if found {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "User with username %s already exists"}`, t.Username)))
return
}
foundUser.Username = t.Username
if foundUser.Role == "" {
foundUser.Role = defaultRole
}
}
if len(t.Tutorial) > 0 {
if !ArrayContains(foundUser.PersonalInfo.Tutorials, t.Tutorial) {
foundUser.PersonalInfo.Tutorials = append(foundUser.PersonalInfo.Tutorials, t.Tutorial)
}
}
if len(t.Firstname) > 0 {
foundUser.PersonalInfo.Firstname = t.Firstname
}
if len(t.Lastname) > 0 {
foundUser.PersonalInfo.Lastname = t.Lastname
}
if len(t.CompanyRole) > 0 {
foundUser.PersonalInfo.Role = t.CompanyRole
}
if project.Environment == "cloud" {
//if len(t.EthInfo.Account) > 0 {
// log.Printf("[DEBUG] Should set ethinfo to %s", t.EthInfo)
// foundUser.EthInfo = t.EthInfo
//}
// Check if UserID is different?
/*
if len(t.UserId) > 0 && t.UserId != foundUser.Id {
log.Printf("[DEBUG] Should set userid to %s", t.UserId)
newUser, err := GetUser(ctx, t.UserId)
if err != nil {
log.Printf("[WARNING] Failed getting user %s: %s", t.UserId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
log.Printf("[DEBUG] Found the user with username %s", newUser.Username)
}
*/
if len(foundUser.PublicProfile.GithubUsername) > 0 {
log.Printf("[DEBUG] Found github username '%s'. User ID to look for: %s", foundUser.PublicProfile.GithubUsername, t.UserId)
}
username := foundUser.PublicProfile.GithubUsername
creator, err := HandleAlgoliaCreatorSearch(ctx, username)
// the same field within Algolia itself.
if err == nil {
// Related to creators
if len(t.CreatorDescription) > 0 {
foundUser.PublicProfile.GithubBio = t.CreatorDescription
}
if len(t.CreatorUrl) > 0 {
foundUser.PublicProfile.GithubUrl = t.CreatorUrl
}
if len(t.CreatorLocation) > 0 {
foundUser.PublicProfile.GithubLocation = t.CreatorLocation
}
if len(t.CreatorSkills) > 0 {
foundUser.PublicProfile.Skills = strings.Split(t.CreatorSkills, ",")
}
if len(t.CreatorWorkStatus) > 0 {
foundUser.PublicProfile.WorkStatus = t.CreatorWorkStatus
}
if len(t.CreatorSocial) > 0 {
foundUser.PublicProfile.Social = strings.Split(t.CreatorSocial, ",")
}
if len(t.SpecializedApps) > 0 {
// FIXME: Update the user in algolia here. Currently just updating existing user
for _, app := range t.SpecializedApps {
found := false
for _, currentApp := range creator.SpecializedApps {
if currentApp.Name == app.Name {
found = true
break
}
}
if !found {
creator.SpecializedApps = append(creator.SpecializedApps, app)
}
}
for _, creatorApp := range creator.SpecializedApps {
// If not found in foundUser.PublicProfile
found := false
for _, userApp := range foundUser.PublicProfile.SpecializedApps {
if userApp.Name == creatorApp.Name {
found = true
break
}
}
if !found {
foundUser.PublicProfile.SpecializedApps = append(foundUser.PublicProfile.SpecializedApps, creatorApp)
}
}
//foundUser.PublicProfile.SpecializedApps = creator.SpecializedApps
}
}
}
if len(t.Suborgs) > 0 && foundUser.Id != userInfo.Id {
//log.Printf("[DEBUG] Got suborg change: %s", t.Suborgs)
// 1. Check if current users' active org is admin in same parent org as user
// 2. Make sure the user should have access to suborg
// 3. Make sure it's ONLY changing orgs based on parent org
// Check which ones the current user has access to
if debug {
log.Printf("[DEBUG] PRE PRE orgs for %s (%s) is len(%d). Input length: %d", foundUser.Username, foundUser.Id, len(foundUser.Orgs), len(t.Suborgs))
}
parentOrgId := userInfo.ActiveOrg.Id
newSuborgs := []string{}
for _, suborg := range t.Suborgs {
if suborg == "REMOVE" {
newSuborgs = append(newSuborgs, suborg)
continue
}
found := false
org, err := GetOrg(ctx, suborg)
if err != nil {
continue
}
if org.CreatorOrg != parentOrgId {
if debug {
log.Printf("[ERROR] Skipping org %s as it is not a suborg of parent org %s for user %s (%s).", suborg, parentOrgId, userInfo.Username, userInfo.Id)
}
continue
}
for _, userOrg := range org.Users {
if userOrg.Id == userInfo.Id {
found = true
break
}
}
if userInfo.SupportAccess && strings.HasSuffix(userInfo.Username, "@shuffler.io") {
found = true
}
if found {
newSuborgs = append(newSuborgs, suborg)
} else {
if debug {
log.Printf("[ERROR] Skipping adding to org %s as user %s (%s) can't edit this one.", suborg, userInfo.Username, userInfo.Id)
}
}
}
t.Suborgs = newSuborgs
addedOrgs := []string{}
for _, suborg := range t.Suborgs {
if suborg == "REMOVE" {
continue
}
if ArrayContains(foundUser.Orgs, suborg) {
continue
}
if !ArrayContains(userInfo.Orgs, suborg) && !userInfo.SupportAccess {
log.Printf("[ERROR] Skipping org %s as user %s (%s) can't edit this one. Should never happen unless direct API usage.", suborg, userInfo.Username, userInfo.Id)
continue
}
foundOrg, err := GetOrg(ctx, suborg)
if err != nil {
if !strings.Contains(err.Error(), "no such entity") {
log.Printf("[WARNING] Failed to get suborg in user edit for %s (%s): %s", foundUser.Username, foundUser.Id, err)
}
continue
}
// Slower but easier :)
parsedOrgs := []string{foundOrg.CreatorOrg}
for _, item := range foundOrg.ManagerOrgs {
parsedOrgs = append(parsedOrgs, item.Id)
}
if !ArrayContains(parsedOrgs, userInfo.ActiveOrg.Id) {
log.Printf("[ERROR] The Org %s (%s) SHOULD NOT BE ADDED for %s (%s): %s. This may indicate a test of the API, as the frontend shouldn't allow it.", foundOrg.Name, suborg, foundUser.Username, foundUser.Id, err)
continue
}
addedOrgs = append(addedOrgs, suborg)
}
// After done, check if ANY of the users' orgs are suborgs of active parent org. If they are, remove.
// Update: This piece runs anyway, in case the job is to REMOVE any suborg
//if len(addedOrgs) > 0 {
//log.Printf("[DEBUG] Orgs to be added: %s. Existing: %s.", addedOrgs, foundUser.Orgs)
// Removed for now due to multi-org chain deleting you from other org chains
if debug {
log.Printf("[DEBUG] Pre orgs for %s (%s) is len(%d)", foundUser.Username, foundUser.Id, len(foundUser.Orgs))
}
newUserOrgs := []string{}
for _, suborg := range foundUser.Orgs {
if suborg == userInfo.ActiveOrg.Id {
newUserOrgs = append(newUserOrgs, suborg)
continue
}
foundOrg, err := GetOrg(ctx, suborg)
if err != nil {
if !strings.Contains(err.Error(), "no such entity") && !strings.Contains(err.Error(), "Org doesn't exist") {
log.Printf("[WARNING] Failed to get suborg in user edit (2) for %s (%s): %s", foundUser.Username, foundUser.Id, err)
}
newUserOrgs = append(newUserOrgs, suborg)
continue
}
// Check if it has anything to do with the parent org, otherwise don't touch it
// CreatorOrg, ManagerOrgs, Suborgs
orgRelevancies := []string{foundOrg.CreatorOrg}
for _, item := range foundOrg.ManagerOrgs {
orgRelevancies = append(orgRelevancies, item.Id)
}
for _, item := range foundOrg.ChildOrgs {
orgRelevancies = append(orgRelevancies, item.Id)
}
if !ArrayContains(orgRelevancies, userInfo.ActiveOrg.Id) {
newUserOrgs = append(newUserOrgs, suborg)
log.Printf("[DEBUG] Org '%s' (%s) is not relevant to parent org %s (%s). Skipping.", foundOrg.Name, foundOrg.Id, userInfo.ActiveOrg.Name, userInfo.ActiveOrg.Id)
continue
}
// Slower but easier :)
parsedOrgs := []string{foundOrg.CreatorOrg}
for _, item := range foundOrg.ManagerOrgs {
parsedOrgs = append(parsedOrgs, item.Id)
}
//if !ArrayContains(parsedOrgs, userInfo.ActiveOrg.Id) {
if !ArrayContains(parsedOrgs, suborg) {
if ArrayContains(t.Suborgs, suborg) {
//log.Printf("[DEBUG] Reappending org %s", suborg)
newUserOrgs = append(newUserOrgs, suborg)
} else {
log.Printf("[DEBUG] Skipping org '%s'", suborg)
}
continue
}
log.Printf("[DEBUG] Should remove user %s (%s) from org %s if it doesn't exist in t.Suborgs", foundUser.Username, foundUser.Id, suborg)
newUsers := []User{}
for _, user := range foundOrg.Users {
if user.Id == foundUser.Id {
continue
}
newUsers = append(newUsers, user)
}
foundOrg.Users = newUsers
err = SetOrg(ctx, *foundOrg, foundOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed setting org when changing user access: %s", err)
}
}
foundUser.Orgs = append(newUserOrgs, addedOrgs...)
log.Printf("[DEBUG] New orgs for %s (%s) is len(%d)", foundUser.Username, foundUser.Id, len(foundUser.Orgs))
}
if len(t.Theme) > 0 && t.Theme != foundUser.Theme {
foundUser.Theme = t.Theme
}
err = SetUser(ctx, foundUser, orgUpdater)
if err != nil {
log.Printf("[WARNING] Error patching user %s: %s", foundUser.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
ip := GetRequestIp(request)
log.Printf("[AUDIT] Successfully updated user %s (%s) with IP: %s", foundUser.Username, foundUser.Id, ip)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func GenerateApikey(ctx context.Context, userInfo User) (User, error) {
if len(userInfo.Username) == 0 || len(userInfo.Id) == 0 {
return userInfo, fmt.Errorf("User ID and username are required to generate API key")
}
// Generate UUID
// Set uuid to apikey in backend (update)
DeleteCache(ctx, fmt.Sprintf("Users_%s", userInfo.ApiKey))
userInfo.ApiKey = uuid.NewV4().String()
err := SetApikey(ctx, userInfo)
if err != nil {
log.Printf("[WARNING] Failed updating apikey: %s", err)
return userInfo, err
}
// Updating user
log.Printf("[INFO] Adding apikey to user '%s'", userInfo.Username)
err = SetUser(ctx, &userInfo, true)
if err != nil {
log.Printf("[WARNING] Failed updating users' apikey: %s", err)
return userInfo, err
}
return userInfo, nil
}
func SetNewWorkflow(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in set new workflow: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to set new workflow: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
var workflow Workflow
err = json.Unmarshal(body, &workflow)
if err != nil {
log.Printf("Failed unmarshaling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Overwriting all settings as a just in case
workflow.ID = uuid.NewV4().String()
workflow.Owner = user.Id
workflow.Sharing = "private"
user.ActiveOrg.Users = []UserMini{}
workflow.ExecutingOrg = user.ActiveOrg
workflow.OrgId = user.ActiveOrg.Id
ctx := GetContext(request)
//err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId)
//if err != nil {
// log.Printf("Failed to increase total workflows stats: %s", err)
//}
if len(workflow.Actions) == 0 {
workflow.Actions = []Action{}
}
if len(workflow.Branches) == 0 {
workflow.Branches = []Branch{}
}
if len(workflow.Triggers) == 0 {
workflow.Triggers = []Trigger{}
}
if len(workflow.Errors) == 0 {
workflow.Errors = []string{}
}
newActions := []Action{}
for _, action := range workflow.Actions {
if action.Environment == "" {
//action.Environment = baseEnvironment
// FIXME: Still necessary? This hinders hybrid mode cloud -> onprem
//if project.Environment == "cloud" {
// action.Environment = "Cloud"
//}
action.IsValid = true
}
//action.LargeImage = ""
newActions = append(newActions, action)
}
// Initialized without functions = adding a hello world node.
if len(newActions) == 0 {
//log.Printf("APPENDING NEW APP FOR NEW WORKFLOW")
// Adds the Testing app if it's a new workflow
workflowapps, err := GetPrioritizedApps(ctx, user)
envName := "cloud"
if project.Environment != "cloud" {
workflowapps, err = GetAllWorkflowApps(ctx, 1000, 0)
envName = "Shuffle"
}
//log.Printf("[DEBUG] Got %d apps. Err: %s", len(workflowapps), err)
if err == nil {
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err == nil {
for _, env := range environments {
if env.Default {
envName = env.Name
break
}
}
}
if workflow.WorkflowAsCode {
// if cloud, activate the app with name Shuffle Tools Fork: 3e320a20966d33c9b7e6790b2705f0bf
if project.Environment == "cloud" {
app, err := GetApp(ctx, "3e320a20966d33c9b7e6790b2705f0bf", user, false)
if err != nil {
log.Printf("[ERROR] Failed getting app: %s", err)
} else {
nodeId := uuid.NewV4().String()
workflow.Start = nodeId
newAction := Action{
Label: "Change Me",
Name: "execute_python",
Environment: envName,
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "call",
Value: "print('Hello world')",
Example: "Repeating: Hello World",
Multiline: true,
},
},
Priority: 0,
Errors: []string{},
ID: nodeId,
IsValid: true,
IsStartNode: true,
Sharing: true,
PrivateID: "",
SmallImage: "",
AppName: app.Name,
AppVersion: app.AppVersion,
AppID: app.ID,
LargeImage: app.LargeImage,
}
newAction.Position = Position{
X: 449.5,
Y: 446.1,
}
newActions = append(newActions, newAction)
}
} else {
// figure out a way to activate Shuffle-Tools-Fork for everyone onprem
}
} else {
for _, item := range workflowapps {
//log.Printf("NAME: %s", item.Name)
if (item.Name == "Shuffle Tools" || item.Name == "Shuffle-Tools") && item.AppVersion == "1.2.0" {
//nodeId := "40447f30-fa44-4a4f-a133-4ee710368737"
nodeId := uuid.NewV4().String()
workflow.Start = nodeId
newAction := Action{
Label: "Change Me",
Name: "repeat_back_to_me",
Environment: envName,
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "call",
Value: "Hello world",
Example: "Repeating: Hello World",
Multiline: true,
},
},
Priority: 0,
Errors: []string{},
ID: nodeId,
IsValid: true,
IsStartNode: true,
Sharing: true,
PrivateID: "",
SmallImage: "",
AppName: item.Name,
AppVersion: item.AppVersion,
AppID: item.ID,
LargeImage: item.LargeImage,
}
newAction.Position = Position{
X: 449.5,
Y: 446.1,
}
newActions = append(newActions, newAction)
break
}
}
}
}
} else {
//log.Printf("[INFO] Has %d actions already", len(newActions))
// FIXME: Check if they require authentication and if they exist locally
//log.Printf("\n\nSHOULD VALIDATE AUTHENTICATION")
//AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"`
//allAuths, err := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
//if err == nil {
// log.Printf("AUTH: %s", allAuths)
// for _, action := range newActions {
// log.Printf("ACTION: %s", action)
// }
//}
}
workflow.Actions = []Action{}
for _, item := range workflow.Actions {
oldId := item.ID
sourceIndexes := []int{}
destinationIndexes := []int{}
for branchIndex, branch := range workflow.Branches {
if branch.SourceID == oldId {
sourceIndexes = append(sourceIndexes, branchIndex)
}
if branch.DestinationID == oldId {
destinationIndexes = append(destinationIndexes, branchIndex)
}
}
item.ID = uuid.NewV4().String()
for _, index := range sourceIndexes {
workflow.Branches[index].SourceID = item.ID
}
for _, index := range destinationIndexes {
workflow.Branches[index].DestinationID = item.ID
}
newActions = append(newActions, item)
}
newTriggers := []Trigger{}
for _, item := range workflow.Triggers {
oldId := item.ID
sourceIndexes := []int{}
destinationIndexes := []int{}
for branchIndex, branch := range workflow.Branches {
if branch.SourceID == oldId {
sourceIndexes = append(sourceIndexes, branchIndex)
}
if branch.DestinationID == oldId {
destinationIndexes = append(destinationIndexes, branchIndex)
}
}
item.ID = uuid.NewV4().String()
for _, index := range sourceIndexes {
workflow.Branches[index].SourceID = item.ID
}
for _, index := range destinationIndexes {
workflow.Branches[index].DestinationID = item.ID
}
item.Status = "uninitialized"
newTriggers = append(newTriggers, item)
}
/*
newSchedules := []Schedule{}
for _, item := range workflow.Schedules {
item.Id = uuid.NewV4().String()
newSchedules = append(newSchedules, item)
}
*/
timeNow := int64(time.Now().Unix())
workflow.Actions = newActions
workflow.Triggers = newTriggers
//workflow.Schedules = newSchedules
workflow.IsValid = true
workflow.Configuration.ExitOnError = false
workflow.Created = timeNow
auth, authOk := request.URL.Query()["set_auth"]
if authOk && len(auth) > 0 && auth[0] == "true" {
allAuths, autherr := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
workflowapps, apperr := GetPrioritizedApps(ctx, user)
if autherr != nil || apperr != nil {
log.Printf("[ERROR] Failed to get auths/app: %s/%s", autherr, apperr)
} else {
for actionIndex, action := range workflow.Actions {
if action.AuthenticationId != "" {
continue
}
// Check if auth is required
outerapp := WorkflowApp{}
for _, app := range workflowapps {
if app.Name != action.AppName {
continue
}
outerapp = app
break
}
if len(outerapp.ID) > 0 && outerapp.Authentication.Required {
for _, auth := range allAuths {
if auth.App.ID == outerapp.ID || auth.App.Name == outerapp.Name {
log.Printf("[DEBUG] Automatically setting authentication for action %s (%s) in workflow %s (%s)", action.Name, action.ID, workflow.Name, workflow.ID)
workflow.Actions[actionIndex].AuthenticationId = auth.Id
}
}
}
}
}
}
workflowjson, err := json.Marshal(workflow)
if err != nil {
log.Printf("Failed workflow json setting marshalling: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
err = SetWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("[WARNING] Failed setting workflow: %s (Set workflow)", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Cleans up cache for the users
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err == nil {
//log.Printf("Getting Org workflows")
workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "")
if err == nil {
updated := false
for tutorialIndex, tutorial := range org.Tutorials {
if tutorial.Name == "Discover Usecases" {
org.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d workflows created. Find more using Workflow Templates or public Workflows.", len(workflows)+1)
if len(workflows) > 0 {
org.Tutorials[tutorialIndex].Done = true
//org.Tutorials[tutorialIndex].Link = "/search?tab=workflows"
//org.Tutorials[tutorialIndex].Link = "/usecases"
org.Tutorials[tutorialIndex].Link = "/welcome?tab=3"
}
updated = true
break
}
}
if updated {
SetOrg(ctx, *org, org.Id)
}
} else {
log.Printf("[ERROR] Failed getting workflows during new workflow creation for updating stats: %s", err)
}
//for _, loopUser := range org.Users {
// cacheKey := fmt.Sprintf("%s_workflows", loopUser.Id)
// DeleteCache(ctx, cacheKey)
//}
} else {
//cacheKey := fmt.Sprintf("%s_workflows", user.Id)
//DeleteCache(ctx, cacheKey)
}
log.Printf("[INFO] Saved new workflow %s with name %s", workflow.ID, workflow.Name)
resp.WriteHeader(200)
resp.Write(workflowjson)
}
func hasBranchChanged(newBranch Branch, oldBranch Branch) (string, bool) {
// Check if there is a difference in parameters, and what they are
if newBranch.Label != oldBranch.Label {
return "label", true
}
if len(newBranch.Conditions) != len(oldBranch.Conditions) {
return "condition_amount", true
}
for _, param := range newBranch.Conditions {
found := false
for _, oldParam := range oldBranch.Conditions {
if param.Condition.ID != oldParam.Condition.ID {
continue
}
if param.Condition.Value != oldParam.Condition.Value {
return "condition_value_type", true
}
if param.Source.Value != oldParam.Source.Value {
return "condition_source", true
}
if param.Destination.Value != oldParam.Destination.Value {
return "condition_destination", true
}
found = true
break
}
if !found {
return "param_not_found", true
}
}
return "", false
}
func hasTriggerChanged(newAction Trigger, oldAction Trigger) (string, bool) {
// Check if there is a difference in parameters, and what they are
if newAction.Name != oldAction.Name {
return "name", true
}
if newAction.Label != oldAction.Label {
return "label", true
}
if newAction.Position.X != oldAction.Position.X || newAction.Position.Y != oldAction.Position.Y {
return "position", true
}
if newAction.AppVersion != oldAction.AppVersion {
return "app_version", true
}
if newAction.IsStartNode != oldAction.IsStartNode {
return "startnode", true
}
for _, param := range newAction.Parameters {
found := false
for _, oldParam := range oldAction.Parameters {
if param.Name != oldParam.Name {
continue
}
if param.Value == oldParam.Value {
// These SHOULD change. That's the whole point.
// We generate a subflow version of the same workflow.
if newAction.TriggerType == "SUBFLOW" && param.Name == "workflow" {
return "param_value", true
}
if newAction.TriggerType == "USERINPUT" && param.Name == "subflow" {
return "param_value", true
}
}
if param.Value != oldParam.Value {
if newAction.TriggerType == "WEBHOOK" {
// Shouldn't change much? Unsure.
} else if newAction.TriggerType == "SUBFLOW" && param.Name == "workflow" {
// We want to check this every time, so subflows are always ran
return "param_value", true
} else if newAction.TriggerType == "USERINPUT" && param.Name == "subflow" {
// We want to check this every time, so subflows are always ran
return "param_value", true
} else {
return "param_value", true
}
}
found = true
break
}
if !found {
log.Printf("[DEBUG] Param not found: %s", param.Name)
return "param_not_found", true
}
}
return "", false
}
func hasActionChanged(newAction Action, oldAction Action) (string, bool) {
// Check if there is a difference in parameters, and what they are
changes := []string{}
if newAction.Name != oldAction.Name {
changes = append(changes, "name")
}
if newAction.Label != oldAction.Label {
changes = append(changes, "label")
}
if newAction.Position.X != oldAction.Position.X || newAction.Position.Y != oldAction.Position.Y {
changes = append(changes, "position")
}
if newAction.AppVersion != oldAction.AppVersion {
changes = append(changes, "app_version")
}
if newAction.AppID != oldAction.AppID {
if debug {
log.Printf("[DEBUG] APPID CHANGED: %s (%#v) vs %s (%#v)", newAction.Name, newAction.AppID, oldAction.Name, oldAction.AppID)
}
changes = append(changes, "app_id")
}
if newAction.IsStartNode != oldAction.IsStartNode {
changes = append(changes, "startnode")
}
if newAction.AuthenticationId != oldAction.AuthenticationId {
changes = append(changes, "authentication_id")
}
if newAction.ExecutionDelay != oldAction.ExecutionDelay {
changes = append(changes, "delay")
}
for _, param := range newAction.Parameters {
found := false
for _, oldParam := range oldAction.Parameters {
if param.Name != oldParam.Name {
continue
}
if param.Value != oldParam.Value {
changes = append(changes, "param_value:"+param.Name)
}
found = true
break
}
if !found {
changes = append(changes, "param_not_found:"+param.Name)
}
}
if len(changes) > 0 {
return strings.Join(changes, ","), true
}
return "", false
}
// Diffs workflows with Child workflows and updates them
func diffWorkflowWrapper(parentWorkflow Workflow) Workflow {
// Actually load the child workflows directly from DB
ctx := context.Background()
childWorkflows, err := ListChildWorkflows(ctx, parentWorkflow.ID)
if err != nil {
return parentWorkflow
}
// Taking care of dedup in case there is a reduction in orgs
newChildWorkflows := []Workflow{}
for _, childWorkflow := range childWorkflows {
if !ArrayContains(parentWorkflow.SuborgDistribution, childWorkflow.OrgId) {
continue
}
newChildWorkflows = append(newChildWorkflows, childWorkflow)
}
newlyAdded := []string{}
childWorkflows = newChildWorkflows
if len(childWorkflows) < len(parentWorkflow.SuborgDistribution) {
for _, suborgId := range parentWorkflow.SuborgDistribution {
found := false
for _, childWorkflow := range childWorkflows {
if childWorkflow.OrgId == suborgId {
found = true
break
}
}
if !found {
//log.Printf("[WARNING] Child workflow of '%s' (%s) for parent org %s may not be distributed to %s yet. Creating or re-finding...", parentWorkflow.Name, parentWorkflow.ID, parentWorkflow.OrgId, suborgId)
childWorkflow, err := GenerateWorkflowFromParent(ctx, parentWorkflow, parentWorkflow.OrgId, suborgId)
if err != nil {
log.Printf("[ERROR] Failed to generate child workflow %s (%s) for %s (%s): %s [diffWorkflowWrapper]", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID, err)
} else {
//log.Printf("[INFO] Generated child workflow %s (%s) for %s (%s)", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID)
childWorkflows = append(childWorkflows, *childWorkflow)
newlyAdded = append(newlyAdded, childWorkflow.ID)
}
}
}
}
//log.Printf("\n\n\nCHILD WORKFLOWS (3): %d\n\n\n", len(childWorkflows))
waitgroup := sync.WaitGroup{}
for _, childWorkflow := range childWorkflows {
// Skipping distrib to old ones~
if !ArrayContains(parentWorkflow.SuborgDistribution, childWorkflow.OrgId) {
continue
}
if len(childWorkflow.Name) == 0 && len(parentWorkflow.ID) == 0 {
continue
}
if len(childWorkflow.ID) == 0 {
continue
}
if childWorkflow.ParentWorkflowId != parentWorkflow.ID {
log.Printf("[WARNING] Child workflow '%s' has a different parent than %s", childWorkflow.ID, parentWorkflow.ID)
continue
}
waitgroup.Add(1)
go func(childWorkflow Workflow, parentWorkflow Workflow, update bool) {
diffWorkflows(childWorkflow, parentWorkflow, true)
// FIXME: This can be heavily optimized by making GenerateWorkflowFromParent() handle more steps itself.
if ArrayContains(newlyAdded, childWorkflow.ID) {
log.Printf("[INFO] Doing a tripple-take on child workflow %s (%s) for %s (%s) during initial setup", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID)
// Reloading it after it's been set
newChildworkflow, err := GetWorkflow(ctx, childWorkflow.ID)
if err != nil {
log.Printf("[WARNING] Failed to get child workflow %s (%s) for %s (%s) during initial setup: %s", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID, err)
} else {
diffWorkflows(*newChildworkflow, parentWorkflow, true)
// Loading it back in
/*
anotherChildWorkflow, err := GetWorkflow(ctx, newChildworkflow.ID)
if err != nil {
log.Printf("[WARNING] Failed to get child workflow %s (%s) for %s (%s) during initial setup (2): %s", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID, err)
} else {
diffWorkflows(*anotherChildWorkflow, parentWorkflow, update)
}
*/
}
}
waitgroup.Done()
}(childWorkflow, parentWorkflow, true)
}
waitgroup.Wait()
return parentWorkflow
}
// Propagates a subflow in a multi-tenant workflow so that
// changes to the subflow also follow the same rules
func subflowDistributionWrapper(parentWorkflow Workflow, childWorkflow Workflow, childTrigger Trigger) Trigger {
//log.Printf("\n\n Calling subflow propagation wrapper for %s (%s) to %s (%s)\n\n", parentWorkflow.Name, parentWorkflow.ID, childWorkflow.Name, childWorkflow.ID)
// This is apparently the parent trigger, and not child
// So now I'm endlessly confused.
trigger := childTrigger
for paramIndex, param := range trigger.Parameters {
if param.Name != "startnode" {
// Use same as action IDs are identical
trigger.Parameters[paramIndex].Value = param.Value
continue
}
if param.Name != "workflow" && param.Name != "subflow" {
continue
}
// since this is an added subflow, the workflow being referred
// is most likely not already distributed. let's do that.
parentSubflowPointedId := param.Value
if len(parentSubflowPointedId) == 0 {
continue
}
if parentSubflowPointedId == parentWorkflow.ID || parentSubflowPointedId == childWorkflow.ID {
log.Printf("[DEBUG] Not distributing workflow '%s' as it's the same as the parent workflow ID", parentSubflowPointedId)
// Point to same
trigger.Parameters[paramIndex].Value = childWorkflow.ID
continue
}
// Check if it's the same as previous revision?
ctx := context.Background()
alreadyPropagatedSubflow := ""
childSubflow, err := GetWorkflow(ctx, parentSubflowPointedId)
if err != nil {
log.Printf("[WARNING] Failed getting parent subflow %s (1): %s", parentSubflowPointedId, err)
continue
}
if childSubflow.OrgId != childWorkflow.OrgId {
log.Printf("[WARNING] Subflow %s is not in the same org as parent workflow %s. This means re-propagation is required (?).", parentSubflowPointedId, parentWorkflow.ID)
} else {
alreadyPropagatedSubflow = childSubflow.ID
}
// childWorkflow vs childSubflow
// Parent workflow ID + Suborg ID = seed
if len(alreadyPropagatedSubflow) > 0 {
//log.Printf("[INFO] Subflow %s (%s) has already been propagated to org %s", childWorkflow.Name, parentSubflowPointedId, childWorkflow.OrgId)
// Just make sure that it now points to that workflow in the Multi-Tenant Workflow
trigger.Parameters[paramIndex].Value = alreadyPropagatedSubflow
workflow, err := GetWorkflow(ctx, alreadyPropagatedSubflow)
if err != nil {
log.Printf("[WARNING] Failed getting propagated subflow: %s", err)
continue
}
if workflow.OrgId != childWorkflow.OrgId {
//log.Printf("[ERROR] Subflow %s has been propagated to %s, but it's not the same org as %s. This means re-propagation is required.", parentSubflowPointedId, childWorkflow.OrgId, childWorkflow.OrgId)
} else {
startNodeIndexToOverwrite := -1
currentStartNode := ""
// taking the right startnode is important
for startNodeIndex, startNode := range trigger.Parameters {
if startNode.Name == "startnode" {
startNodeIndexToOverwrite = startNodeIndex
currentStartNode = startNode.Value
}
}
if len(currentStartNode) == 0 {
continue
}
for _, action := range workflow.Actions {
if action.ID == currentStartNode {
trigger.Parameters[startNodeIndexToOverwrite].Value = action.ID
break
}
}
continue
}
}
// Getting the PARENT workflow
parentSubflowPointed, err := GetWorkflow(ctx, parentSubflowPointedId)
if err != nil {
log.Printf("[WARNING] Failed getting parent subflow %s (2): %s", parentSubflowPointedId, err)
continue
}
// Propagates ALL the relevant ID's at once to avoid desync from goroutines
if !ArrayContains(parentSubflowPointed.SuborgDistribution, childWorkflow.OrgId) {
for _, parentTenantId := range parentWorkflow.SuborgDistribution {
if !ArrayContains(parentSubflowPointed.SuborgDistribution, parentTenantId) {
log.Printf("[DEBUG] Adding org %s to subflow %s (%s)", parentTenantId, parentSubflowPointed.Name, parentSubflowPointed.ID)
parentSubflowPointed.SuborgDistribution = append(parentSubflowPointed.SuborgDistribution, parentTenantId)
}
}
}
err = SetWorkflow(ctx, *parentSubflowPointed, parentSubflowPointedId)
if err != nil {
log.Printf("[WARNING] Failed setting parent subflow: %s", err)
continue
}
log.Printf("[DEBUG] Re-propagating subflow %s (%s) to %s (%s)", parentSubflowPointed.Name, parentSubflowPointed.ID, childWorkflow.Name, childWorkflow.ID)
propagatedSubflow, err := GenerateWorkflowFromParent(ctx, *parentSubflowPointed, parentSubflowPointed.OrgId, childWorkflow.OrgId)
if err != nil {
log.Printf("[ERROR] Failed to generate child workflow %s (%s) for %s (%s): %s [subflowDistributionWrapper]", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID, err)
// This means it will point to the parent. What do we do?
trigger.Parameters[paramIndex].Value = ""
} else {
trigger.Parameters[paramIndex].Value = propagatedSubflow.ID
}
startnode := ""
startNodeParamIndex := -1
// now handle startnode
for startNodeParamIndex_, param_ := range trigger.Parameters {
if param_.Name == "startnode" {
startnode = param_.Value
startNodeParamIndex = startNodeParamIndex_
}
}
if len(startnode) == 0 {
continue
}
// actions are always startnodes
// find the equivalent of the startnode in the new workflow
for _, action := range propagatedSubflow.Actions {
if action.ID == startnode {
trigger.Parameters[startNodeParamIndex].Value = action.ID
break
}
}
}
return trigger
}
func deleteScheduleGeneral(ctx context.Context, scheduleId string) error {
schedule, err := GetSchedule(ctx, scheduleId)
if err != nil {
log.Printf("[WARNING] Failed getting schedule %s: %s", scheduleId, err)
return err
}
if project.Environment == "cloud" && (schedule.Environment == "" || schedule.Environment == "cloud") {
log.Printf("[INFO] Deleting schedule with ID %s", scheduleId)
c, err := scheduler.NewCloudSchedulerClient(ctx)
if err != nil {
log.Printf("[WARNING] Failed deleting %s", err)
return err
}
defaultLocation := os.Getenv("SHUFFLE_GCE_LOCATION")
req := &schedulerpb.DeleteJobRequest{
Name: fmt.Sprintf("projects/%s/locations/%s/jobs/schedule_%s", gceProject, defaultLocation, scheduleId),
}
log.Printf("[INFO] Request made to GCP to delete schedule %s", scheduleId)
err = c.DeleteJob(ctx, req)
if err != nil {
//log.Printf("[WARNING] Failed deleting cloud schedule %s", err)
return err
}
log.Printf("[INFO] Deleted schedule with ID %s", scheduleId)
err = DeleteKey(ctx, "schedules", scheduleId)
if err != nil {
log.Printf("[WARNING] Failed deleting schedule %s locally: %s", scheduleId, err)
return err
}
} else if project.Environment == "onprem" && (schedule.Environment == "onprem" || schedule.Environment == "") {
// TODO: to be handled
} else if project.Environment == "cloud" && schedule.Environment == "onprem" {
// hybrid case
// TODO: to be handled
} else if project.Environment == "onprem" && (schedule.Environment == "cloud") {
scheduleWorkflow, err := GetWorkflow(ctx, schedule.WorkflowId)
if err != nil {
log.Printf("[WARNING] Failed getting schedule workflow %s: %s", schedule.WorkflowId, err)
return err
}
org, err := GetOrg(ctx, schedule.Org)
if err != nil {
log.Printf("Failed finding org %s: %s", org.Id, err)
return err
}
// 1. Send request to cloud
// 2. Remove schedule if success
action := CloudSyncJob{
Type: "schedule",
Action: "stop",
OrgId: org.Id,
PrimaryItemId: scheduleId,
SecondaryItem: schedule.Frequency,
ThirdItem: scheduleWorkflow.ID,
}
err = executeCloudAction(action, org.SyncConfig.Apikey)
if err != nil {
log.Printf("[WARNING] Failed cloud action STOP schedule: %s", err)
return err
} else {
log.Printf("[INFO] Successfully ran cloud action STOP schedule")
err = DeleteKey(ctx, "schedules", scheduleId)
if err != nil {
log.Printf("[WARNING] Failed deleting schedule %s locally: %s", scheduleId, err)
return err
}
}
}
return nil
}
// This is the main function that handles the diffing
// and merging of workflows in multi-tenant environments
func diffWorkflows(oldWorkflow Workflow, parentWorkflow Workflow, update bool) {
// Check if there is a difference in actions, and what they are
// Check if there is a difference in triggers, and what they are
// Check if there is a difference in branches, and what they are
//log.Printf("[DEBUG] PRE Child workflow %s. Actions: %d, Triggers: %d, Branches: %d", oldWorkflow.ID, len(oldWorkflow.Actions), len(oldWorkflow.Triggers), len(oldWorkflow.Branches))
// We create a new ID for each trigger.
// Older ID is stored in trigger.ReplacementForTrigger
ctx := context.Background()
nameChanged := false
descriptionChanged := false
tagsChanged := false
backupsChanged := false
inputfieldsChanged := false
discoveredEnvironment := ""
addedActions := []string{}
removedActions := []string{}
updatedActions := []Action{}
addedTriggers := []string{}
removedTriggers := []string{}
updatedTriggers := []Trigger{}
addedBranches := []string{}
removedBranches := []string{}
updatedBranches := []Branch{}
if oldWorkflow.Name != parentWorkflow.Name {
nameChanged = true
}
if oldWorkflow.Description != parentWorkflow.Description {
descriptionChanged = true
}
if oldWorkflow.BackupConfig.UploadRepo != parentWorkflow.BackupConfig.UploadRepo || oldWorkflow.BackupConfig.UploadBranch != parentWorkflow.BackupConfig.UploadBranch || oldWorkflow.BackupConfig.UploadUsername != parentWorkflow.BackupConfig.UploadUsername || oldWorkflow.BackupConfig.UploadToken != parentWorkflow.BackupConfig.UploadToken {
backupsChanged = true
}
if len(oldWorkflow.Tags) != len(parentWorkflow.Tags) {
tagsChanged = true
}
if len(oldWorkflow.InputQuestions) != len(parentWorkflow.InputQuestions) {
inputfieldsChanged = true
}
// Child workflow env & auth id mapping
parentWorkflowEnvironment := "cloud"
if project.Environment != "cloud" {
parentWorkflowEnvironment = "Shuffle"
}
for _, action := range parentWorkflow.Actions {
if len(action.Environment) > 0 {
parentWorkflowEnvironment = action.Environment
break
}
}
oldWorkflowEnvs, err := GetEnvironments(ctx, oldWorkflow.OrgId)
if err != nil {
log.Printf("[ERROR][%s] Failed to get distributed workflow environments: %s", oldWorkflow.OrgId, err)
}
// Keep the environment unchanged for the
// distributed workflow if the parent workflow runtime enviroment
// does not exist.
for _, action := range oldWorkflow.Actions {
// Change all the distributed workflow to cloud if
// parent workflow runtime changes to cloud.
if strings.ToLower(parentWorkflowEnvironment) == "cloud" {
discoveredEnvironment = parentWorkflowEnvironment
break
}
for _, env := range oldWorkflowEnvs {
if strings.ToLower(parentWorkflowEnvironment) != "cloud" && parentWorkflowEnvironment == env.Name && action.Environment != parentWorkflowEnvironment {
discoveredEnvironment = parentWorkflowEnvironment
break
} else {
discoveredEnvironment = action.Environment
}
}
}
if len(discoveredEnvironment) == 0 {
discoveredEnvironment = parentWorkflowEnvironment
}
for _, newAction := range parentWorkflow.Actions {
found := false
if !newAction.ParentControlled {
continue
}
for _, oldAction := range oldWorkflow.Actions {
if !oldAction.ParentControlled {
continue
}
if newAction.ID == oldAction.ID {
found = true
break
}
}
if !found {
addedActions = append(addedActions, newAction.ID)
}
}
for _, oldAction := range oldWorkflow.Actions {
found := false
if !oldAction.ParentControlled {
continue
}
for _, newAction := range parentWorkflow.Actions {
if !newAction.ParentControlled {
continue
}
if oldAction.ID == newAction.ID {
found = true
break
}
}
if !found {
removedActions = append(removedActions, oldAction.ID)
}
}
for _, parentAction := range parentWorkflow.Actions {
if !parentAction.ParentControlled {
continue
}
if ArrayContains(addedActions, parentAction.ID) || ArrayContains(removedActions, parentAction.ID) {
continue
}
for _, oldAction := range oldWorkflow.Actions {
if !oldAction.ParentControlled {
continue
}
if parentAction.ID != oldAction.ID {
continue
}
changeType, changed := hasActionChanged(parentAction, oldAction)
if changed || len(changeType) > 0 {
if debug {
log.Printf("[DEBUG] Action %s (%s) has changed in '%s'", parentAction.Label, parentAction.ID, changeType)
}
updatedActions = append(updatedActions, parentAction)
}
}
}
// Triggers
for _, parentTrigger := range parentWorkflow.Triggers {
if !parentTrigger.ParentControlled {
continue
}
found := false
for _, childTrigger := range oldWorkflow.Triggers {
if childTrigger.ReplacementForTrigger == parentTrigger.ID {
found = true
break
}
if parentTrigger.ID == childTrigger.ID {
found = true
break
}
seedString := fmt.Sprintf("%s_%s", parentTrigger.ID, oldWorkflow.ID)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
comparisonString := uuid.Must(uuid.FromBytes(uuidBytes)).String()
if childTrigger.ID == comparisonString {
found = true
break
}
}
if !found {
//log.Printf("[WARNING] Trigger %s (%s) has been added.", parentTrigger.Label, parentTrigger.ID)
// If status is running & this is webhook, start them on the fly
addedTriggers = append(addedTriggers, parentTrigger.ID)
}
}
// Checks if parentWorkflow removed a trigger
// that was distributed to child workflow.
for _, childTrigger := range oldWorkflow.Triggers {
if !childTrigger.ParentControlled && len(childTrigger.ReplacementForTrigger) == 0 {
continue
}
found := false
for _, parentTrigger := range parentWorkflow.Triggers {
if childTrigger.ReplacementForTrigger == parentTrigger.ID {
found = true
break
}
if parentTrigger.ID == childTrigger.ID {
found = true
break
}
// Static ID, so this may work if ID mapping is wrong somewhere
seedString := fmt.Sprintf("%s_%s", parentTrigger.ID, oldWorkflow.ID)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
comparisonString := uuid.Must(uuid.FromBytes(uuidBytes)).String()
if childTrigger.ID == comparisonString {
found = true
break
}
}
if !found {
//log.Printf("[WARNING] Trigger %s (%s) could not be found anymore? (%#v). Parentcontrolled: %#v", childTrigger.Label, childTrigger.ID, childTrigger.ReplacementForTrigger, childTrigger.ParentControlled)
removedTriggers = append(removedTriggers, childTrigger.ID)
}
}
// fun. newAction is from parentWorkflow, of course.
// and oldAction is from child. This can get confusing!
for _, parentTrigger := range parentWorkflow.Triggers {
if ArrayContains(addedTriggers, parentTrigger.ID) || ArrayContains(removedTriggers, parentTrigger.ID) {
continue
}
for _, childTrigger := range oldWorkflow.Triggers {
if childTrigger.ReplacementForTrigger != parentTrigger.ID {
continue
}
_, changed := hasTriggerChanged(parentTrigger, childTrigger)
if changed {
//log.Printf("[DEBUG] Trigger %s (%s) has changed in '%s'", parentTrigger.Label, parentTrigger.ID, changeType)
updatedTriggers = append(updatedTriggers, parentTrigger)
}
}
}
// Branches
for _, newBranch := range parentWorkflow.Branches {
if !newBranch.ParentControlled {
continue
}
found := false
for _, oldBranch := range oldWorkflow.Branches {
if !oldBranch.ParentControlled {
continue
}
if newBranch.ID == oldBranch.ID {
found = true
break
}
}
if !found {
addedBranches = append(addedBranches, newBranch.ID)
}
}
for _, oldBranch := range oldWorkflow.Branches {
if !oldBranch.ParentControlled {
continue
}
found := false
for _, newBranch := range parentWorkflow.Branches {
if !newBranch.ParentControlled {
continue
}
if oldBranch.ID == newBranch.ID {
found = true
break
}
}
if !found {
removedBranches = append(removedBranches, oldBranch.ID)
}
}
for _, newBranch := range parentWorkflow.Branches {
if !newBranch.ParentControlled {
//log.Printf("SKIP1: %#v", newBranch)
continue
}
if ArrayContains(addedBranches, newBranch.ID) || ArrayContains(removedBranches, newBranch.ID) {
//log.Printf("SKIP2: %#v", newBranch)
continue
}
// Verifies a ton of stuff about branches to ensure they are
// kept synced, even with e.g. ACTION/TRIGGER ID changes
for oldBranchIndex, oldBranch := range oldWorkflow.Branches {
if !oldBranch.ParentControlled {
continue
}
if newBranch.ID != oldBranch.ID {
continue
}
// Finding if e.g. an ID is found or not
foundSource := false
foundDestination := false
for _, action := range oldWorkflow.Actions {
if action.ID == newBranch.SourceID {
foundSource = true
continue
}
if action.ID == newBranch.DestinationID {
foundDestination = true
}
}
for _, trigger := range oldWorkflow.Triggers {
if trigger.ID == newBranch.SourceID {
foundSource = true
continue
}
if trigger.ID == newBranch.DestinationID {
foundDestination = true
}
}
if !foundSource || !foundDestination {
//log.Printf("[ERROR] Branch %s in workflow %s is missing something. Source: %s (%#v), Dest: %s (%#v)", newBranch.ID, oldWorkflow.ID, newBranch.SourceID, foundSource, newBranch.DestinationID, foundDestination)
// Loop through source & destination + triggers and find if the seed version exists or not
if !foundSource {
seedString := fmt.Sprintf("%s_%s", newBranch.SourceID, oldWorkflow.ID)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
newSource := uuid.Must(uuid.FromBytes(uuidBytes)).String()
// Check triggers if it exists, as actions should not be changing ID
for _, trigger := range oldWorkflow.Triggers {
if trigger.ID == newSource {
foundSource = true
oldWorkflow.Branches[oldBranchIndex].SourceID = newSource
break
}
}
}
if !foundDestination {
seedString := fmt.Sprintf("%s_%s", newBranch.DestinationID, oldWorkflow.ID)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
newSource := uuid.Must(uuid.FromBytes(uuidBytes)).String()
// Check triggers if it exists, as actions should not be changing ID
for _, trigger := range oldWorkflow.Triggers {
if trigger.ID == newSource {
foundSource = true
oldWorkflow.Branches[oldBranchIndex].DestinationID = newSource
break
}
}
}
oldBranch = oldWorkflow.Branches[oldBranchIndex]
}
changeType, changed := hasBranchChanged(newBranch, oldBranch)
if changed {
_ = changeType
log.Printf("[DEBUG] Branch %s (%s) has changed in '%s'", newBranch.Label, newBranch.ID, changeType)
updatedBranches = append(updatedBranches, newBranch)
}
}
}
// Create / Delete / Modify tracking
lastParentRevision := Workflow{}
parentRevisions, err := ListWorkflowRevisions(ctx, parentWorkflow.ID, 2)
if err != nil {
log.Printf("[WARNING] Failed getting parent revisions: %s", err)
} else {
if len(parentRevisions) > 0 {
lastParentRevision = parentRevisions[0]
}
}
if update {
// FIXME: This doesn't work does it?
childWorkflow := oldWorkflow
if parentWorkflow.OrgId == childWorkflow.OrgId {
log.Printf("[ERROR] Parent and child orgs are the same for workflow %s (%s). This is not possible during multi tenant distribution and is most likely a bug somewhere.", childWorkflow.Name, childWorkflow.ID)
childWorkflow.Errors = append(childWorkflow.Errors, "Parent and child orgs are the same for workflow %s.", childWorkflow.Name)
return
}
if len(childWorkflow.SuborgDistribution) > 0 {
log.Printf("[ERROR] Disabled suborg distribution for child workflow %s (%s). This usually only happens due to an ID bug somewhere.", childWorkflow.Name, childWorkflow.ID)
childWorkflow.Errors = append(childWorkflow.Errors, "Suborg distribution disabled automatically in child workflow %s.", childWorkflow.Name)
childWorkflow.SuborgDistribution = []string{}
}
// log.Printf("\n\nSTART")
//log.Printf("[DEBUG] CHILD ACTIONS START: %d", len(childWorkflow.Actions))
//log.Printf("[DEBUG] CHILD TRIGGERS START: %d", len(childWorkflow.Triggers))
//log.Printf("[DEBUG] CHILD BRANCHES START: %d\n\n", len(childWorkflow.Branches))
if nameChanged {
childWorkflow.Name = parentWorkflow.Name
}
if descriptionChanged {
childWorkflow.Description = parentWorkflow.Description
}
if tagsChanged {
childWorkflow.Tags = parentWorkflow.Tags
}
if backupsChanged {
childWorkflow.BackupConfig = parentWorkflow.BackupConfig
}
if inputfieldsChanged {
childWorkflow.InputQuestions = parentWorkflow.InputQuestions
}
// Check variables and directly change them
for _, parentVariable := range parentWorkflow.WorkflowVariables {
found := false
for childIndex, childVariable := range childWorkflow.WorkflowVariables {
if parentVariable.Name == childVariable.Name {
if childVariable.Value != parentVariable.Value {
relevantRevisionVariable := Variable{}
for _, parentRevisionVariable := range lastParentRevision.WorkflowVariables {
if parentRevisionVariable.Name == parentVariable.Name {
relevantRevisionVariable = parentRevisionVariable
break
}
}
if relevantRevisionVariable.Value == childVariable.Value {
childWorkflow.WorkflowVariables[childIndex].Value = parentVariable.Value
}
}
found = true
break
}
}
if !found {
childWorkflow.WorkflowVariables = append(childWorkflow.WorkflowVariables, parentVariable)
}
}
// Check variables and directly change them
for _, parentVariable := range parentWorkflow.ExecutionVariables {
found := false
for childIndex, childVariable := range childWorkflow.ExecutionVariables {
if parentVariable.Name == childVariable.Name {
if childVariable.Value != parentVariable.Value {
relevantRevisionVariable := Variable{}
for _, parentRevisionVariable := range lastParentRevision.ExecutionVariables {
if parentRevisionVariable.Name == parentVariable.Name {
relevantRevisionVariable = parentRevisionVariable
break
}
}
if relevantRevisionVariable.Value == childVariable.Value {
childWorkflow.ExecutionVariables[childIndex].Value = parentVariable.Value
}
}
found = true
break
}
}
if !found {
childWorkflow.ExecutionVariables = append(childWorkflow.ExecutionVariables, parentVariable)
}
}
childActions := []Action{}
for _, action := range oldWorkflow.Actions {
// Check if it SHOULD be parent controlled
for _, newAction := range parentWorkflow.Actions {
if newAction.ID == action.ID {
action.ParentControlled = true
break
}
}
if action.ParentControlled {
continue
}
childActions = append(childActions, action)
}
childTriggers := []Trigger{}
for _, trigger := range oldWorkflow.Triggers {
for _, newTrigger := range parentWorkflow.Triggers {
if newTrigger.ID == trigger.ID {
trigger.ParentControlled = true
break
}
}
if trigger.ParentControlled {
continue
}
// those of which aren't parent controlled triggers,
// are added to childTriggers.
childTriggers = append(childTriggers, trigger)
}
childBranches := []Branch{}
for _, branch := range oldWorkflow.Branches {
for _, newBranch := range parentWorkflow.Branches {
if newBranch.ID == branch.ID {
branch.ParentControlled = true
break
}
}
if branch.ParentControlled {
continue
}
childBranches = append(childBranches, branch)
}
if len(addedActions) > 0 {
actions := childActions
for _, action := range parentWorkflow.Actions {
if !ArrayContains(addedActions, action.ID) {
continue
}
actions = append(actions, action)
}
childWorkflow.Actions = append(childWorkflow.Actions, actions...)
childActions = childWorkflow.Actions
}
if len(removedActions) > 0 {
newChildActions := childActions
for _, action := range childWorkflow.Actions {
if ArrayContains(removedActions, action.ID) {
continue
}
newChildActions = append(newChildActions, action)
}
childWorkflow.Actions = newChildActions
childActions = childWorkflow.Actions
}
var err error
childAuths := []AppAuthenticationStorage{}
if len(childAuths) == 0 {
childAuths, err = GetAllWorkflowAppAuth(ctx, childWorkflow.OrgId)
if err != nil {
log.Printf("[WARNING] Failed getting auths for child org %s: %s", childWorkflow.OrgId, err)
}
}
// FIXME: Not necessary in the future, but useful for now
// Makes sure we double check EVERY node
if len(updatedActions) == 0 {
updatedActions = parentWorkflow.Actions
}
if len(updatedActions) > 0 {
for _, action := range updatedActions {
for childIndex, childAction := range childWorkflow.Actions {
if childAction.ID != action.ID {
// this means it's a new action
continue
}
childWorkflow.Actions[childIndex].Environment = discoveredEnvironment
childWorkflow.Actions[childIndex].ParentControlled = true
// This has the PREVIOUS value of the current workflow, as to diff if the parent itself has changed at all.
relevantRevisionAction := Action{}
for _, parentRevisionAction := range lastParentRevision.Actions {
if parentRevisionAction.ID == action.ID {
relevantRevisionAction = parentRevisionAction
break
}
}
if childAction.Label != action.Label {
//log.Printf("[DEBUG] Updating label in child action '%s'", childAction.ID)
childWorkflow.Actions[childIndex].Label = action.Label
}
if childAction.AppID != action.AppID {
childWorkflow.Actions[childIndex].AppID = action.AppID
}
if childAction.AppName != action.AppName {
childWorkflow.Actions[childIndex].AppName = action.AppName
}
if childAction.AppVersion != action.AppVersion {
childWorkflow.Actions[childIndex].AppVersion = action.AppVersion
}
if childAction.Name != action.Name {
//log.Printf("[DEBUG] Updating action in child action '%s'", childAction.ID)
// Override entirely?
childWorkflow.Actions[childIndex].Name = action.Name
childWorkflow.Actions[childIndex].Parameters = action.Parameters
childWorkflow.Actions[childIndex].LargeImage = action.LargeImage
childAction.Parameters = childWorkflow.Actions[childIndex].Parameters
}
// FIXME:
// Make sure it changes:
// auth, env
// name, app_version, app_id, app_name
// startnode
// execution delay
// parameters
// position
if action.Position.X != childAction.Position.X || action.Position.Y != childAction.Position.Y {
//log.Printf("[DEBUG] Position has changed. Updating in child action '%s'", childAction.ID)
childWorkflow.Actions[childIndex].Position = action.Position
}
if action.IsStartNode && !childAction.IsStartNode {
//log.Printf("[DEBUG] Updating start node in child action '%s'", childAction.ID)
// Check if the startnode is any of the parent nodes. If it is, then we change. If it is not in the parent nodes, we don't change.
foundInParentWorkflow := false
for _, parentActionInner := range parentWorkflow.Actions {
if parentActionInner.ID != action.ID {
continue
}
foundInParentWorkflow = true
break
}
// If it is found in the parent workflow, we change it. Otherwise it's most likely a local override
if foundInParentWorkflow {
//log.Printf("[DEBUG] Updating start node in child action '%s'", childAction.ID)
childWorkflow.Start = action.ID
childWorkflow.Actions[childIndex].IsStartNode = true
childAction.IsStartNode = true
}
}
if action.ExecutionDelay != childAction.ExecutionDelay {
//log.Printf("[DEBUG] Updating delay in child action '%s'", childAction.ID)
childWorkflow.Actions[childIndex].ExecutionDelay = action.ExecutionDelay
}
if len(action.AuthenticationId) > 0 && len(childAction.AuthenticationId) == 0 {
// Check if the auth is available or not, in case it's distributed.
for _, childAuth := range childAuths {
if childAuth.Id != action.AuthenticationId {
continue
}
//log.Printf("[DEBUG] Updating auth in child as it is available")
childWorkflow.Actions[childIndex].AuthenticationId = action.AuthenticationId
break
}
}
// FIXME: Use the last revision to check the previous value of the param
for _, parentParam := range action.Parameters {
for childParamIndex, childParam := range childAction.Parameters {
if parentParam.Name != childParam.Name {
continue
}
// FIXME: Track if the value is the same as the OLD parent value, or if it has changed. If it's the same as the OLD value, we don't change it.
relevantRevisionActionParam := WorkflowAppActionParameter{}
for _, parentRevisionAction := range relevantRevisionAction.Parameters {
if parentRevisionAction.Name != parentParam.Name {
continue
}
relevantRevisionActionParam = parentRevisionAction
break
}
// Checks if the previous value of the parent workflow is the same as the child, as to keep in sync
if relevantRevisionActionParam.Value != parentParam.Value {
if childParam.Value == relevantRevisionActionParam.Value {
childWorkflow.Actions[childIndex].Parameters[childParamIndex].Value = parentParam.Value
}
}
if len(childWorkflow.Actions[childIndex].Parameters) <= childParamIndex {
log.Printf("[ERROR] Child action %s has more params than parent %s in child workflow '%s'. This should ONLY happen if an app is updated directly", childAction.ID, action.ID, childWorkflow.ID)
break
}
if len(parentParam.Value) > 0 && len(childWorkflow.Actions[childIndex].Parameters[childParamIndex].Value) == 0 {
log.Printf("[DEBUG] Updating param %s in child action '%s'", parentParam.Name, childAction.ID)
childWorkflow.Actions[childIndex].Parameters[childParamIndex].Value = parentParam.Value
}
if parentParam.Value != childWorkflow.Actions[childIndex].Parameters[childParamIndex].Value {
//log.Printf("[DEBUG] Param %s in child action '%s' has changed", parentParam.Name, childAction.ID)
// FIXME: Find out if it's a local change in the child workflow or not
}
break
}
}
break
}
}
}
replacedTriggers := []string{}
for _, trigger := range oldWorkflow.Triggers {
if len(trigger.ReplacementForTrigger) > 0 {
replacedTriggers = append(replacedTriggers, trigger.ReplacementForTrigger)
}
}
if len(addedTriggers) > 0 {
// the case where a new trigger is
// added to a previously distributed workflow
triggers := childTriggers
for _, trigger := range parentWorkflow.Triggers {
//log.Printf("[DEBUG] MAYBE added trigger: %#v", trigger.ID)
if !ArrayContains(addedTriggers, trigger.ID) {
continue
}
if ArrayContains(replacedTriggers, trigger.ID) {
continue
}
// change ID, and replace in branch ID
seedString := fmt.Sprintf("%s_%s", trigger.ID, childWorkflow.ID)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
oldID := trigger.ID
trigger.ReplacementForTrigger = trigger.ID
trigger.ID = uuid.Must(uuid.FromBytes(uuidBytes)).String()
trigger.ParentControlled = true
//log.Printf("[DEBUG] Adding new node. Old ID: %s, New ID: %s", oldID, trigger.ID)
if trigger.TriggerType == "SCHEDULE" {
// FIXME: Have their own trigger or nah?
} else if trigger.TriggerType == "WEBHOOK" {
parentHook, err := GetHook(ctx, oldID)
if err != nil {
log.Printf("[ERROR] Parent hook load error: %#v", err)
} else {
trigger.Status = parentHook.Status
}
foundUrl := ""
auth := ""
customResponse := ""
version := "v1"
for paramIndex, param := range trigger.Parameters {
if param.Name == "url" {
foundUrl = param.Value
trigger.Parameters[paramIndex].Value = strings.Replace(param.Value, fmt.Sprintf("webhook_%s", oldID), fmt.Sprintf("webhook_%s", trigger.ID), -1)
}
if param.Name == "tmp" {
trigger.Parameters[paramIndex].Value = fmt.Sprintf("webhook_%s", trigger.ID)
}
if param.Name == "auth_headers" {
auth = param.Value
}
if param.Name == "custom_response_body" {
customResponse = param.Value
}
if param.Name == "await_response" {
version = param.Value
}
}
if trigger.Status == "running" {
startNode := ""
for _, branch := range parentWorkflow.Branches {
if branch.SourceID == oldID {
startNode = branch.DestinationID
break
}
}
hook := Hook{
Id: trigger.ID,
Start: startNode,
Workflows: []string{childWorkflow.ID},
Info: Info{
Name: trigger.Name,
Description: trigger.Label,
Url: foundUrl,
},
Type: "webhook",
Owner: childWorkflow.OrgId,
Actions: []HookAction{
HookAction{
Type: "workflow",
Name: childWorkflow.Name,
Id: childWorkflow.ID,
Field: "",
},
},
OrgId: childWorkflow.OrgId,
Environment: trigger.Environment,
Auth: auth,
CustomResponse: customResponse,
Version: version,
VersionTimeout: 15,
Running: true,
Status: "running",
}
err = SetHook(ctx, hook)
if err != nil {
log.Printf("[ERROR] Failed setting hook in child workflow: %s", err)
}
}
} else if trigger.TriggerType == "SUBFLOW" {
// params: workflow, argument, user_apikey, startnode,
// check_result and auth_override
trigger = subflowDistributionWrapper(parentWorkflow, childWorkflow, trigger)
} else if trigger.TriggerType == "USERINPUT" {
log.Printf("[DEBUG] User input trigger added: %#v", trigger.ID)
trigger = subflowDistributionWrapper(parentWorkflow, childWorkflow, trigger)
}
for branchIndex, branch := range childWorkflow.Branches {
if branch.SourceID == oldID {
childWorkflow.Branches[branchIndex].SourceID = trigger.ID
}
if branch.DestinationID == oldID {
childWorkflow.Branches[branchIndex].DestinationID = trigger.ID
}
}
triggers = append(triggers, trigger)
}
childWorkflow.Triggers = append(childWorkflow.Triggers, triggers...)
childTriggers = childWorkflow.Triggers
}
if len(removedTriggers) > 0 {
//log.Printf("[DEBUG] Removed triggers: %#v. CHILD: %d", removedTriggers, len(childTriggers))
//newChildTriggers := childTriggers
newChildTriggers := []Trigger{}
for _, trigger := range childWorkflow.Triggers {
if !ArrayContains(removedTriggers, trigger.ID) {
// Just making sure it exists
newChildTriggers = append(newChildTriggers, trigger)
continue
}
// while removing triggers,
// make sure to stop them as well
// need to handle this better
// Q: is there a generic API that we can call
// to have this handled?
if trigger.TriggerType == "WEBHOOK" {
ctx := context.Background()
hook, err := GetHook(ctx, trigger.ID)
if err == nil && hook.OrgId == childWorkflow.OrgId {
// this anyhow, means it is a webhook
err = DeleteKey(ctx, "hooks", hook.Id)
if err != nil {
log.Printf("[WARNING] Failed deleting hook: %s", err)
}
continue
}
log.Printf("[WARNING] Failed getting child hook: %s", err)
continue
} else if trigger.TriggerType == "SCHEDULE" {
//log.Printf("[DEBUG] This trigger is a schedule. Will proceed to delete it")
ctx := context.Background()
schedule, err := GetSchedule(ctx, trigger.ID)
if err == nil {
err = deleteScheduleGeneral(ctx, schedule.Id)
if err != nil {
log.Printf("[WARNING] Failed deleting schedule: %s", err)
}
continue
}
log.Printf("[ERROR] Failed getting child schedule: %s", err)
continue
} else if trigger.TriggerType == "SUBFLOW" {
//log.Printf("[DEBUG] This trigger is a subflow. Will proceed to delete it")
continue
} else if trigger.TriggerType == "USERINPUT" {
//log.Printf("[DEBUG] This trigger is a user input. Will proceed to delete it")
continue
} else if trigger.TriggerType == "PIPELINE" {
//log.Printf("[DEBUG] This trigger is a pipeline. Will proceed to delete it")
continue
}
// This breaks the whole thing about removing triggers
//newChildTriggers = append(newChildTriggers, trigger)
}
childWorkflow.Triggers = newChildTriggers
childTriggers = childWorkflow.Triggers
}
if len(updatedTriggers) > 0 {
// UpdatedTriggers = list of parent triggers
for _, parentTrigger := range updatedTriggers {
//log.Printf("[DEBUG] ID of the parent trigger (%s): %s", parentTrigger.TriggerType, parentTrigger.ID)
for childIndex, childTrigger := range childWorkflow.Triggers {
if childTrigger.ReplacementForTrigger != parentTrigger.ID {
continue
}
if parentTrigger.Status == "SUCCESS" {
//log.Printf("[DEBUG] Remapping parent status SUCCESS to running for child trigger %s", childTrigger.ID)
parentTrigger.Status = "running"
}
// Ensures params are in sync, at least with the size of them
if len(childTrigger.Parameters) != len(parentTrigger.Parameters) {
//log.Printf("[WARNING] Re-syncing parameters in child trigger with parent trigger %s", childTrigger.Name)
childWorkflow.Triggers[childIndex].Parameters = parentTrigger.Parameters
}
relevantRevisionTrigger := Trigger{}
for _, parentRevisionTrigger := range lastParentRevision.Triggers {
if parentRevisionTrigger.ID == parentTrigger.ID {
relevantRevisionTrigger = parentRevisionTrigger
break
}
}
// Check for desynced parameters
for _, parentParam := range parentTrigger.Parameters {
for childParamIndex, childParam := range childTrigger.Parameters {
if parentParam.Name != childParam.Name {
continue
}
if parentParam.Value == childParam.Value {
// No point in checking stuff then
continue
}
relevantRevisionTriggerParam := WorkflowAppActionParameter{}
for _, parentRevisionParam := range relevantRevisionTrigger.Parameters {
if parentRevisionParam.Name != parentParam.Name {
continue
}
relevantRevisionTriggerParam = parentRevisionParam
break
}
// Checks if the previous value of the parent workflow is the same as the child, as to keep in sync
if relevantRevisionTriggerParam.Value != parentParam.Value {
if childParam.Value == relevantRevisionTriggerParam.Value {
childWorkflow.Triggers[childIndex].Parameters[childParamIndex].Value = parentParam.Value
} else {
//log.Printf("[DEBUG] NOT SAME: %s != %s", childParam.Value, relevantRevisionTriggerParam.Value)
// Parent workflow ID changed to not match.. Need to check if the ID is a seeded version of the same ID
if childParam.Name == "workflow" || childParam.Name == "subflow" {
// Check if seed(relevantRevisionTriggerParam.Value) == childParam.Value as to see if parent has changed, but child is in sync
seedString := fmt.Sprintf("%s_%s", relevantRevisionTriggerParam.Value, childWorkflow.OrgId)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
newId := uuid.Must(uuid.FromBytes(uuidBytes)).String()
// Means the seed is the same, and we should change as parent is changing
if newId == childParam.Value {
childWorkflow.Triggers[childIndex].Parameters[childParamIndex].Value = parentParam.Value
}
}
}
}
}
}
// FIXME:
// Make sure it changes things such as URL & references properly
if childTrigger.TriggerType == "WEBHOOK" {
log.Printf("[DEBUG] Updating webhook trigger %s", childTrigger.ID)
// make sure to only override: name, label, position,
// app_version, startnode and nothing else
childWorkflow.Triggers[childIndex].Name = parentTrigger.Name
childWorkflow.Triggers[childIndex].Label = parentTrigger.Label
childWorkflow.Triggers[childIndex].Position = parentTrigger.Position
childWorkflow.Triggers[childIndex].AppVersion = parentTrigger.AppVersion
// 1. Get the parent ID
// 2. Check it's still running or not
parentHook, err := GetHook(ctx, parentTrigger.ID)
if err != nil {
log.Printf("[ERROR] Parent hook load error: %#v", err)
} else {
parentTrigger.Status = parentHook.Status
}
childWorkflow.Triggers[childIndex].Status = parentTrigger.Status
if parentTrigger.Status != childWorkflow.Status {
log.Printf("[DEBUG] Webhook: Status change in trigger %#v compared to parent. Parent: %#v, Child: %#v", childWorkflow.Triggers[childIndex].ID, parentTrigger.Status, childWorkflow.Status)
if parentTrigger.Status == "running" {
// Start the trigger
log.Printf("[DEBUG] Starting trigger child %s", childTrigger.ID)
parentHook, err := GetHook(ctx, parentTrigger.ID)
if err != nil {
log.Printf("[ERROR] Parent hook load error: %#v", err)
} else {
childHook := parentHook
childHook.Id = childTrigger.ID
childHook.Workflows = []string{childWorkflow.ID}
childHook.Owner = childWorkflow.OrgId
childHook.OrgId = childWorkflow.OrgId
childHook.Status = "running"
childHook.Running = true
err = SetHook(ctx, *childHook)
if err != nil {
log.Printf("[ERROR] Failed setting hook in child workflow update (2): %s", err)
}
}
} else {
log.Printf("[DEBUG] Stopping webhook trigger child %s", childTrigger.ID)
err = DeleteKey(ctx, "hooks", childTrigger.ID)
if err != nil {
log.Printf("[WARNING] Failed deleting hook: %s", err)
}
}
}
break
} else if parentTrigger.TriggerType == "SCHEDULE" {
// app_version and parameters
childWorkflow.Triggers[childIndex].Name = parentTrigger.Name
childWorkflow.Triggers[childIndex].Label = parentTrigger.Label
childWorkflow.Triggers[childIndex].Position = parentTrigger.Position
childWorkflow.Triggers[childIndex].AppVersion = parentTrigger.AppVersion
childWorkflow.Triggers[childIndex].Status = parentTrigger.Status
for paramIndex, param := range parentTrigger.Parameters {
if param.Name == "execution_argument" {
childWorkflow.Triggers[childIndex].Parameters[paramIndex].Value = param.Value
}
if param.Name == "cron" {
childWorkflow.Triggers[childIndex].Parameters[paramIndex].Value = param.Value
}
}
log.Printf("[DEBUG] Updating schedule trigger %s", childTrigger.ID)
break
} else if parentTrigger.TriggerType == "SUBFLOW" {
// make sure to override: name, label, position,
// app_version, startnode and parameters
childWorkflow.Triggers[childIndex].Name = parentTrigger.Name
childWorkflow.Triggers[childIndex].Label = parentTrigger.Label
childWorkflow.Triggers[childIndex].Position = parentTrigger.Position
childWorkflow.Triggers[childIndex].AppVersion = parentTrigger.AppVersion
// essentially, now we try to verify:
// okay, new workflow? we see it's a subflow that's
childWorkflow.Triggers[childIndex] = subflowDistributionWrapper(parentWorkflow, childWorkflow, childWorkflow.Triggers[childIndex])
break
} else if parentTrigger.TriggerType == "USERINPUT" {
// make sure to override: name, label, position,
// app_version, startnode and parameters
childWorkflow.Triggers[childIndex].Name = parentTrigger.Name
childWorkflow.Triggers[childIndex].Label = parentTrigger.Label
childWorkflow.Triggers[childIndex].Position = parentTrigger.Position
childWorkflow.Triggers[childIndex].AppVersion = parentTrigger.AppVersion
childWorkflow.Triggers[childIndex] = subflowDistributionWrapper(parentWorkflow, childWorkflow, childWorkflow.Triggers[childIndex])
break
} else if parentTrigger.TriggerType == "PIPELINE" {
childWorkflow.Triggers[childIndex].Name = parentTrigger.Name
childWorkflow.Triggers[childIndex].Label = parentTrigger.Label
childWorkflow.Triggers[childIndex].Position = parentTrigger.Position
childWorkflow.Triggers[childIndex].AppVersion = parentTrigger.AppVersion
childWorkflow.Triggers[childIndex].Parameters = parentTrigger.Parameters
log.Printf("[DEBUG] Updating pipeline trigger %s", childTrigger.ID)
break
}
childWorkflow.Triggers[childIndex] = parentTrigger
break
}
}
}
if len(addedBranches) > 0 {
branches := childBranches
for _, branch := range parentWorkflow.Branches {
if !ArrayContains(addedBranches, branch.ID) {
continue
}
// if a new branch is added to add a trigger,
// make sure it has the new trigger ID
for _, trigger := range childWorkflow.Triggers {
if trigger.ReplacementForTrigger == branch.SourceID {
branch.SourceID = trigger.ID
} else if trigger.ReplacementForTrigger == branch.DestinationID {
branch.DestinationID = trigger.ID
}
}
branches = append(branches, branch)
}
childWorkflow.Branches = append(childWorkflow.Branches, branches...)
childBranches = childWorkflow.Branches
}
if len(removedBranches) > 0 {
newChildBranches := childBranches
for _, branch := range childWorkflow.Branches {
if ArrayContains(removedBranches, branch.ID) {
continue
}
newChildBranches = append(newChildBranches, branch)
}
childWorkflow.Branches = newChildBranches
childBranches = childWorkflow.Branches
}
if len(updatedBranches) > 0 {
for _, action := range updatedBranches {
for index, childAction := range childWorkflow.Branches {
if childAction.ID != action.ID {
continue
}
childWorkflow.Branches[index] = action
break
}
}
}
// Dedup actions, triggers & branches
newActions := []Action{}
newTriggers := []Trigger{}
newBranches := []Branch{}
for childActionIndex, childAction := range childWorkflow.Actions {
// Check if the parent workflow has it, and make sure parent controlled is set
childWorkflow.Actions[childActionIndex].Environment = discoveredEnvironment
for _, newAction := range parentWorkflow.Actions {
if newAction.ID == childAction.ID {
newAction.ParentControlled = true
childWorkflow.Actions[childActionIndex].ParentControlled = true
break
}
}
// the below authentication overwriting doesn't work.
idFound := false
for _, oldWorkflowAction := range oldWorkflow.Actions {
if oldWorkflowAction.ID == childAction.ID {
idFound = true
childWorkflow.Actions[childActionIndex].AuthenticationId = oldWorkflowAction.AuthenticationId
}
}
if !idFound {
for _, oldWorkflowAction := range oldWorkflow.Actions {
if oldWorkflowAction.AppID == childAction.AppID {
childWorkflow.Actions[childActionIndex].AuthenticationId = oldWorkflowAction.AuthenticationId
break
}
}
}
found := false
for _, newAction := range newActions {
if newAction.ID == childAction.ID {
found = true
continue
}
}
// looks like a hack stitched together
// only to make sure to never miss action.
if !found {
newActions = append(newActions, childAction)
}
}
for childTriggerIndex, childTrigger := range childWorkflow.Triggers {
childWorkflow.Triggers[childTriggerIndex].Environment = discoveredEnvironment
for _, newTrigger := range parentWorkflow.Triggers {
if newTrigger.ID == childTrigger.ID {
childTrigger.ParentControlled = true
childWorkflow.Triggers[childTriggerIndex].ParentControlled = true
break
}
}
found := false
for _, newTrigger := range newTriggers {
if newTrigger.ID == childTrigger.ID {
found = true
continue
}
}
if !found {
newTriggers = append(newTriggers, childTrigger)
}
}
for childBranchIndex, childBranch := range childWorkflow.Branches {
for _, newBranch := range parentWorkflow.Branches {
if newBranch.ID == childBranch.ID || (newBranch.SourceID == childBranch.SourceID && newBranch.DestinationID == childBranch.DestinationID) {
childBranch.ParentControlled = true
childWorkflow.Branches[childBranchIndex].ParentControlled = true
break
}
}
found := false
for _, newBranch := range newBranches {
if newBranch.ID == childBranch.ID {
found = true
continue
}
}
if !found {
newBranches = append(newBranches, childBranch)
}
}
childWorkflow.Actions = newActions
childWorkflow.Triggers = newTriggers
childWorkflow.Branches = newBranches
// Update the org with all the relevant apps and doing it before health check
childOrg, err := GetOrg(ctx, childWorkflow.OrgId)
if err != nil {
log.Printf("[ERROR] Failed to load multi-tenant workflow org %s: %s", childWorkflow.OrgId, err)
} else {
oldLength := len(childOrg.ActiveApps)
handled := []string{}
for _, action := range childWorkflow.Actions {
if ArrayContains(handled, action.AppID) {
continue
}
found := false
for _, appId := range childOrg.ActiveApps {
if appId == action.AppID {
found = true
break
}
}
if !found {
childOrg.ActiveApps = append(childOrg.ActiveApps, action.AppID)
}
}
if len(childOrg.ActiveApps) > oldLength {
err := SetOrg(ctx, *childOrg, childOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed updating child org %s during multi-tenant workflow update: %s", childOrg.Name, err)
}
}
}
//log.Printf("[DEBUG] CHILD ACTIONS END: %d", len(childWorkflow.Actions))
//log.Printf("[DEBUG] CHILD TRIGGERS END: %d", len(childWorkflow.Triggers))
//log.Printf("[DEBUG] CHILD BRANCHES END: %d\n\n", len(childWorkflow.Branches))
childWorkflow, _, err = GetStaticWorkflowHealth(ctx, childWorkflow)
if err != nil {
log.Printf("[ERROR] Failed getting static workflow health for %s: %s", childWorkflow.ID, err)
}
err = SetWorkflow(ctx, childWorkflow, childWorkflow.ID)
if err != nil {
log.Printf("[ERROR] Failed updating child workflow %s from parent workflow %s: %s", childWorkflow.ID, oldWorkflow.ID, err)
} else {
//log.Printf("[INFO] Updated child workflow '%s' based on parent %s", childWorkflow.ID, oldWorkflow.ID)
SetWorkflowRevision(ctx, childWorkflow)
passedOrg := Org{
Id: childWorkflow.ExecutingOrg.Id,
Name: childWorkflow.ExecutingOrg.Name,
}
SetGitWorkflow(ctx, childWorkflow, &passedOrg)
}
go DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", oldWorkflow.ID))
go DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", childWorkflow.ID))
go DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", parentWorkflow.ID))
}
}
// Saves a workflow to an ID
func SaveWorkflow(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in save workflow: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to save workflow (2): %s (%s)", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
}
if len(fileId) != 36 {
log.Printf(`[WARNING] Workflow ID %s is not valid`, fileId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID to save is not valid"}`))
return
}
// Here to check access rights
ctx := GetContext(request)
tmpworkflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Failed getting the workflow %s locally (save workflow): %s", fileId, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
workflow := Workflow{}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed workflow body read: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
err = json.Unmarshal([]byte(body), &workflow)
if err != nil {
//log.Printf(string(body))
log.Printf("[ERROR] Failed workflow unmarshaling (save): %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
// This fix the region issues with public workflow but can it create problem?
if len(tmpworkflow.ID) == 0 || workflow.Public == true {
log.Printf("[WARNING] Failed to find public workflow in region, using user provided workflow data")
tmp := workflow
tmpworkflow = &tmp
}
if project.Environment == "cloud" && tmpworkflow.Validated == false {
if workflow.Validated == true {
if !user.SupportAccess {
workflow.Validated = false
} else {
//log.Printf("[INFO] User %s is validating workflow %s", user.Username, tmpworkflow.ID)
}
}
}
type PublicCheck struct {
UserEditing bool `json:"user_editing"`
Public bool `json:"public"`
Owner string `json:"owner"`
}
/*
if len(workflow.ParentWorkflowId) > 0 || len(tmpworkflow.ParentWorkflowId) > 0 {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Can't save a workflow distributed from your parent org"}`))
return
}
*/
if len(workflow.InputQuestions) > 0 {
log.Printf("[DEBUG] Making ALL '%d' input questions required for workflow %s", len(workflow.InputQuestions), workflow.ID)
}
for qIndex, _ := range workflow.InputQuestions {
workflow.InputQuestions[qIndex].Required = true
}
isPartner := false
correctUser := false
if user.Id != tmpworkflow.Owner || tmpworkflow.Public == true {
log.Printf("[AUDIT] User %s is accessing workflow %s (save workflow)", user.Username, tmpworkflow.ID)
// if,ifelse: Public, Org owns it, or user owns it
if tmpworkflow.Public {
// FIXME:
// If the user Id is part of the creator: DONT update this way.
// /users/creators/username
// Just making sure
if project.Environment == "cloud" {
//algoliaUser, err := HandleAlgoliaCreatorSearch(ctx, username)
// First case : Check if the user's active org is Partner
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting your org details"}`))
return
}
if org.LeadInfo.TechPartner || org.LeadInfo.IntegrationPartner || org.LeadInfo.DistributionPartner || org.LeadInfo.ServicePartner || org.LeadInfo.ChannelPartner {
isPartner = true
}
// Check if the partner org owns the workflow and user is an admin
if isPartner && (len(workflow.Owner) > 0 && workflow.Owner == user.ActiveOrg.Id && user.Role == "admin") {
log.Printf("[INFO] User %s (%s) is saving a public workflow %s as a partner org", user.Username, user.Id, workflow.ID)
correctUser = true
tmpworkflow.Public = true
workflow.Public = true
// Check if the user is the owns the workflow
} else if len(workflow.Owner) > 0 && workflow.Owner == user.Id {
log.Printf("[INFO] User %s (%s) is saving a public workflow %s", user.Username, user.Id, workflow.ID)
correctUser = true
tmpworkflow.Public = true
workflow.Public = true
} else {
// Check if the user is allowed to edit the public workflow (Algolia Creator search)
algoliaUser, err := HandleAlgoliaCreatorSearch(ctx, user.PublicProfile.GithubUsername)
if err != nil {
// Using the Allowlist to check if the user is allowed to edit the public workflow
allowList := os.Getenv("GITHUB_USER_ALLOWLIST")
log.Printf("[WARNING] User with ID %s for Workflow %s could not be found (workflow update): %s. Username: %s. ACL controlled with GITHUB_USER_ALLOWLIST environment variable. Allowed users: %#v", user.Id, tmpworkflow.ID, err, user.PublicProfile.GithubUsername, allowList)
// Check if current user is one of the few allowed
// This can only happen if the workflow doesn't already have an owner
if user.PublicProfile.Public && len(allowList) > 0 {
allowListSplit := strings.Split(allowList, ",")
for _, username := range allowListSplit {
if username != user.PublicProfile.GithubUsername {
continue
}
algoliaUser, err = HandleAlgoliaCreatorSearch(ctx, user.PublicProfile.GithubUsername)
if err != nil {
log.Printf("[ERROR] Algolia Creator search error in public workflow edit: %s", err)
continue
}
break
}
}
}
wf2 := PublicCheck{}
err = json.Unmarshal([]byte(body), &wf2)
if err != nil {
log.Printf("[ERROR] Failed workflow unmarshaling (save - 2): %s", err)
}
if algoliaUser.ObjectID == user.Id || ArrayContains(algoliaUser.Synonyms, user.Id) {
log.Printf("[WARNING] User %s (%s) has access to edit %s! Keep it public!!", user.Username, user.Id, workflow.ID)
// Means the owner is using the workflow for their org
if wf2.UserEditing == false {
correctUser = false
} else {
correctUser = true
tmpworkflow.Public = true
workflow.Public = true
}
}
}
}
// FIX: Should check if this workflow has already been saved?
if !correctUser {
log.Printf("[INFO] User %s is saving the public workflow %s", user.Username, tmpworkflow.ID)
workflow = *tmpworkflow
workflow.PublishedId = workflow.ID
workflow.ID = uuid.NewV4().String()
workflow.Public = false
workflow.Owner = user.Id
workflow.Org = []OrgMini{
user.ActiveOrg,
}
workflow.ExecutingOrg = user.ActiveOrg
workflow.OrgId = user.ActiveOrg.Id
workflow.PreviouslySaved = false
newTriggers := []Trigger{}
changedIds := map[string]string{}
for _, trigger := range workflow.Triggers {
newId := uuid.NewV4().String()
trigger.Environment = "cloud"
hookAuth := ""
customResponse := ""
for paramIndex, param := range trigger.Parameters {
if param.Name == "url" {
trigger.Parameters[paramIndex].Value = fmt.Sprintf("https://shuffler.io/api/v1/hooks/webhook_%s", newId)
}
if param.Name == "auth_headers" {
hookAuth = param.Value
}
if param.Name == "custom_response_body" {
customResponse = param.Value
}
}
if trigger.TriggerType != "SCHEDULE" {
trigger.Status = "running"
if trigger.TriggerType == "WEBHOOK" {
hook := Hook{
Id: newId,
Start: workflow.Start,
Workflows: []string{workflow.ID},
Info: Info{
Name: trigger.Name,
Description: trigger.Description,
Url: fmt.Sprintf("https://shuffler.io/api/v1/hooks/webhook_%s", newId),
},
Type: "webhook",
Owner: user.Username,
Status: "running",
Actions: []HookAction{
HookAction{
Type: "workflow",
Name: trigger.Name,
Id: workflow.ID,
Field: "",
},
},
Running: true,
OrgId: workflow.OrgId,
Environment: "cloud",
Auth: hookAuth,
CustomResponse: customResponse,
}
log.Printf("[DEBUG] Starting hook %s for user %s (%s) during Workflow Save for %s", hook.Id, user.Username, user.Id, workflow.ID)
err = SetHook(ctx, hook)
if err != nil {
log.Printf("[WARNING] Failed setting hook during workflow copy of %s: %s", workflow.ID, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
}
changedIds[trigger.ID] = newId
trigger.ID = newId
//log.Printf("New id for %s: %s", trigger.TriggerType, trigger.ID)
newTriggers = append(newTriggers, trigger)
}
newBranches := []Branch{}
for _, branch := range workflow.Branches {
for key, value := range changedIds {
if branch.SourceID == key {
branch.SourceID = value
}
if branch.DestinationID == key {
branch.DestinationID = value
}
}
newBranches = append(newBranches, branch)
}
workflow.Branches = newBranches
workflow.Triggers = newTriggers
err = SetWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("[WARNING] Failed saving NEW version of public %s for user %s: %s", tmpworkflow.ID, user.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org for cache release for public wf: %s", err)
} else {
for _, loopUser := range org.Users {
//DeleteCache(ctx, fmt.Sprintf("%s_workflows", org.Id)
//DeleteCache(ctx, fmt.Sprintf("%s_workflows", loopUser.Id))
//DeleteCache(ctx, fmt.Sprintf("apps_%s", loopUser.Id))
//DeleteCache(ctx, fmt.Sprintf("apps_%s", org.Id)
DeleteCache(ctx, fmt.Sprintf("user_%s", loopUser.Id))
}
// Activate all that aren't already there
changed := false
for _, action := range workflow.Actions {
//log.Printf("App: %s, Public: %s", action.AppID, action.Public)
if !ArrayContains(org.ActiveApps, action.AppID) {
org.ActiveApps = append(org.ActiveApps, action.AppID)
changed = true
}
}
if changed {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[ERROR] Failed updating active app list for org %s (%s): %s", org.Name, org.Id, err)
} else {
//DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
//DeleteCache(ctx, fmt.Sprintf("apps_%s", org.Id))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-100"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-500"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-1000"))
DeleteCache(ctx, "all_apps")
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
}
}
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "new_id": "%s"}`, workflow.ID)))
return
}
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
// Re-added this as in most cases when our users or customers need help, it makes it
// so we can finalize the workflow for them
log.Printf("[AUDIT] Letting verified support admin %s access workflow %s (save workflow)", user.Username, workflow.ID)
workflow.ID = tmpworkflow.ID
} else if tmpworkflow.OrgId == user.ActiveOrg.Id && user.Role != "org-reader" {
log.Printf("[AUDIT] User %s is accessing workflow %s (save workflow)", user.Username, tmpworkflow.ID)
workflow.ID = tmpworkflow.ID
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (save)", user.Username, tmpworkflow.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Wrong user for workflow. Do you have write access?"}`))
return
}
} else {
log.Printf("[AUDIT] User %s is creating or modifying workflow with ID %s as they are the owner OR it's public. Actions: %d, Triggers: %d", user.Username, workflow.ID, len(workflow.Actions), len(workflow.Triggers))
if workflow.Public {
log.Printf("[WARNING] Rolling back public as the user set it to true themselves")
workflow.Public = false
}
if len(workflow.PublishedId) > 0 {
log.Printf("[INFO] Workflow %s has the published ID %s", workflow.ID, workflow.PublishedId)
// Overwrite ID here?
}
}
referer := request.Header.Get("Referer")
if !strings.Contains(referer, "/forms/") {
workflow.Sharing = tmpworkflow.Sharing
workflow.InputQuestions = tmpworkflow.InputQuestions
workflow.FormControl = tmpworkflow.FormControl
}
if fileId != workflow.ID {
log.Printf("[ERROR] Path and request ID are NOT matching in workflow save: %s != %s. URL: %s", fileId, workflow.ID, request.URL.String())
resp.WriteHeader(400)
//resp.Write([]byte(`{"success": false, "reason": "ID in workflow data and path are not matching"}`))
resp.Write([]byte(`{"success": false, "reason": "ID in workflow data and path are not matching. Export and re-import this workflow for use in your region."}`))
return
}
if len(workflow.Name) == 0 {
log.Printf("[WARNING] Can't save workflow without a name.")
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow needs a name"}`))
return
}
if len(workflow.Actions) == 0 {
log.Printf("[WARNING] Can't save a workflow without a single action.")
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow needs at least one action"}`))
return
}
newsuborgs := []string{}
for _, suborg := range workflow.SuborgDistribution {
if len(suborg) != 36 {
continue
}
newsuborgs = append(newsuborgs, suborg)
}
// This is an autofixer for variables in actions
for i, _ := range workflow.Actions {
workflow.Actions[i].SourceExecution = ""
workflow.Actions[i].SourceWorkflow = ""
}
workflow.SuborgDistribution = newsuborgs
if len(workflow.SuborgDistribution) != len(tmpworkflow.SuborgDistribution) {
log.Printf("[AUDIT] Suborg distribution changed by user %s (%s) for workflow %s (%s) in org %s (%s). Clearing cache for suborgs.", user.Username, user.Id, workflow.Name, workflow.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
// Clear workflow cache
for _, suborg := range workflow.SuborgDistribution {
cacheKey := fmt.Sprintf("%s_workflows", suborg)
DeleteCache(ctx, cacheKey)
}
for _, suborg := range tmpworkflow.SuborgDistribution {
cacheKey := fmt.Sprintf("%s_workflows", suborg)
DeleteCache(ctx, cacheKey)
}
}
// Resetting subflows as they shouldn't be entirely saved. Used just for imports/exports only
if len(workflow.Subflows) > 0 {
log.Printf("[DEBUG] Got %d subflows saved in %s (to be saved and removed)", len(workflow.Subflows), workflow.ID)
for _, subflow := range workflow.Subflows {
go SetWorkflow(ctx, subflow, subflow.ID)
}
workflow.Subflows = []Workflow{}
}
if strings.ToLower(workflow.Status) == "test" {
workflow.Status = "test"
} else if strings.ToLower(workflow.Status) == "prod" {
workflow.Status = "production"
} else {
if len(workflow.Status) == 0 {
workflow.Status = "test"
}
// Custom statuses allowed with API
if len(workflow.Status) > 255 {
workflow.Status = workflow.Status[:255]
}
}
workflow.Subflows = []Workflow{}
if len(workflow.DefaultReturnValue) > 0 && len(workflow.DefaultReturnValue) < 200 {
log.Printf("[INFO] Set default return value to on failure to (%s): %s", workflow.ID, workflow.DefaultReturnValue)
//workflow.DefaultReturnValue
}
//log.Printf("[INFO] Saving workflow '%s' with %d action(s) and %d trigger(s). Org: %s", workflow.Name, len(workflow.Actions), len(workflow.Triggers), workflow.OrgId)
if len(workflow.OrgId) == 0 && len(user.ActiveOrg.Id) > 0 {
if len(workflow.ExecutingOrg.Id) == 0 {
log.Printf("[INFO] Setting executing org for workflow to %s", user.ActiveOrg.Id)
user.ActiveOrg.Users = []UserMini{}
workflow.ExecutingOrg = user.ActiveOrg
}
if len(workflow.OrgId) == 0 {
workflow.OrgId = user.ActiveOrg.Id
}
} else if len(workflow.OrgId) != 0 && len(workflow.ExecutingOrg.Id) == 0 {
log.Printf("[INFO] Setting executing org for workflow to %s", workflow.OrgId)
workflow.ExecutingOrg.Id = workflow.OrgId
workflow.ExecutingOrg.Name = ""
}
orgUpdated := false
workflow.Categories = Categories{}
if workflow.OrgId == "" {
workflow.OrgId = user.ActiveOrg.Id
}
workflow, allNodes, err := GetStaticWorkflowHealth(ctx, workflow)
if err != nil {
log.Printf("[ERROR] Failed getting static workflow health for %s: %s", workflow.ID, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting static workflow health: %s"}`, err.Error())))
return
}
// Nodechecks
foundNodes := []string{}
for _, node := range allNodes {
for _, branch := range workflow.Branches {
if node == branch.DestinationID || node == branch.SourceID {
foundNodes = append(foundNodes, node)
break
}
// Check if source parent
}
}
if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 {
// This shit takes a few seconds lol
if !workflow.IsValid {
oldworkflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Workflow %s doesn't exist - oldworkflow.", fileId)
if workflow.PreviouslySaved {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Item already exists."}`))
return
}
}
oldworkflow.IsValid = false
err = SetWorkflow(ctx, *oldworkflow, fileId)
if err != nil {
log.Printf("[WARNING] Failed saving workflow to database: %s", err)
if workflow.PreviouslySaved {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow already saved before"}`))
return
}
}
}
}
workflowapps, apperr := GetPrioritizedApps(ctx, user)
if apperr != nil {
log.Printf("[ERROR] Failed getting apps for org %s", user.ActiveOrg.Id)
}
newActions := []Action{}
allAuths, autherr := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
if !workflow.PreviouslySaved {
log.Printf("[WARNING] WORKFLOW INIT FOR %s: NOT PREVIOUSLY SAVED - SET ACTION AUTH!", workflow.ID)
timeNow := int64(time.Now().Unix())
//workflow.ID = uuid.NewV4().String()
// Get the workflow and check if we own it
skipRebuild := false
newWorkflow, err := GetWorkflow(ctx, workflow.ID)
if err == nil && newWorkflow.OrgId == user.ActiveOrg.Id {
skipRebuild = true
workflow.PreviouslySaved = true
} else if err != nil || len(newWorkflow.Actions) != 1 {
log.Printf("[ERROR] FAILED GETTING WORKFLOW: %s - CREATING NEW ID!", err)
workflow.ID = uuid.NewV4().String()
}
if !skipRebuild {
workflow.Public = false
workflow.Owner = user.Id
workflow.ExecutingOrg = user.ActiveOrg
workflow.OrgId = user.ActiveOrg.Id
workflow.Created = timeNow
workflow.Edited = timeNow
workflow.Org = []OrgMini{
user.ActiveOrg,
}
if autherr == nil && len(workflowapps) > 0 && apperr == nil {
//log.Printf("Setting actions")
actionFixing := []Action{}
appsAdded := []string{}
for _, action := range workflow.Actions {
setAuthentication := false
if len(action.AuthenticationId) > 0 {
//found := false
authenticationFound := false
for _, auth := range allAuths {
if auth.Id == action.AuthenticationId {
authenticationFound = true
break
}
}
if !authenticationFound {
setAuthentication = true
}
} else {
// FIXME: 1. Validate if the app needs auth
// 1. Validate if auth for the app exists
// var appAuth AppAuthenticationStorage
setAuthentication = true
}
if setAuthentication {
authSet := false
for _, auth := range allAuths {
if !auth.Active {
continue
}
if !auth.Defined {
continue
}
if auth.App.Name == action.AppName {
//log.Printf("FOUND AUTH FOR APP %s: %s", auth.App.Name, auth.Id)
action.AuthenticationId = auth.Id
authSet = true
break
}
}
// FIXME: Only o this IF there isn't another one for the app already
if !authSet {
//log.Printf("Validate if the app NEEDS auth or not")
outerapp := WorkflowApp{}
for _, app := range workflowapps {
if app.Name == action.AppName {
outerapp = app
break
}
}
if len(outerapp.ID) > 0 && outerapp.Authentication.Required {
found := false
for _, auth := range allAuths {
if auth.App.ID == outerapp.ID {
found = true
break
}
}
for _, added := range appsAdded {
if outerapp.ID == added {
found = true
}
}
_ = found
log.Printf("[DEBUG] NOT adding authentication for workflow %s (%s) in org %s (%s) automatically as this should be done from within the workflow/during setup.", workflow.Name, workflow.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
action.Errors = append(action.Errors, "Requires authentication")
action.IsValid = false
workflow.IsValid = false
}
}
}
actionFixing = append(actionFixing, action)
}
newActions = actionFixing
} else {
log.Printf("FirstSave error: %s - %s", err, apperr)
//allAuths, err := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
}
skipSave, skipSaveOk := request.URL.Query()["skip_save"]
if skipSaveOk && len(skipSave) > 0 {
//log.Printf("INSIDE SKIPSAVE: %s", skipSave[0])
if strings.ToLower(skipSave[0]) != "true" {
workflow.PreviouslySaved = true
}
} else {
workflow.PreviouslySaved = true
}
}
workflow.UpdatedBy = ""
workflow.Errors = []string{}
workflow.Validation = TypeValidation{}
}
if len(newActions) > 1 {
workflow.Actions = newActions
}
auth, authOk := request.URL.Query()["set_auth"]
if authOk && len(auth) > 0 && auth[0] == "true" {
for actionIndex, action := range workflow.Actions {
if action.AuthenticationId != "" {
continue
}
// Check if auth is required
outerapp := WorkflowApp{}
for _, app := range workflowapps {
if app.Name != action.AppName {
continue
}
outerapp = app
break
}
if len(outerapp.ID) > 0 && outerapp.Authentication.Required {
for _, auth := range allAuths {
if auth.App.ID == outerapp.ID || auth.App.Name == outerapp.Name {
log.Printf("[DEBUG] Automatically setting authentication for action %s (%s) in workflow %s (%s)", action.Name, action.ID, workflow.Name, workflow.ID)
workflow.Actions[actionIndex].AuthenticationId = auth.Id
}
}
}
}
}
workflow.IsValid = true
// TBD: Is this too drastic? May lead to issues in the future.
if workflow.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] NOT Editing workflow to be owned by org %s. Instead just editing. Original org: %s", user.ActiveOrg.Id, workflow.OrgId)
/*
workflow.OrgId = user.ActiveOrg.Id
workflow.ExecutingOrg = user.ActiveOrg
workflow.Org = append(workflow.Org, user.ActiveOrg)
*/
//resp.WriteHeader(500)
//resp.Write([]byte(`{"success": false, "error": "Workflow does not belong to this org"}`))
//return
}
// Only happens if the workflow is public and being edited
if correctUser {
workflow.Public = true
if isPartner && (len(workflow.Owner) > 0 && workflow.Owner != user.Id) {
workflow.Owner = user.ActiveOrg.Id
} else {
workflow.Owner = user.Id
}
// Should save it in Algolia too?
_, err = handleAlgoliaWorkflowUpdate(ctx, workflow)
if err != nil {
log.Printf("[ERROR] Failed finding publicly changed workflow %s for user %s (%s): %s", workflow.ID, user.Username, user.Id, err)
} else {
log.Printf("[DEBUG] User %s (%s) updated their public workflow %s (%s)", user.Username, user.Id, workflow.Name, workflow.ID)
}
}
if len(workflow.SuborgDistribution) > 0 {
if len(workflow.ParentWorkflowId) > 0 {
// In case they are
log.Printf("[ERROR] User %s (%s) tried to save %s with BOTH parent and child workflow distribution. Removing suborg distribution. Most likely frontend desync.", user.Username, user.Id, workflow.ID)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Can't both be a parent and child workflow at the same time. Please remove suborg distribution."}`))
return
}
//log.Printf("[DEBUG] Diffing based on parent workflow %s", workflow.ID)
for actionIndex, _ := range workflow.Actions {
workflow.Actions[actionIndex].ParentControlled = true
}
for triggerIndex, _ := range workflow.Triggers {
workflow.Triggers[triggerIndex].ParentControlled = true
}
for branchIndex, _ := range workflow.Branches {
workflow.Branches[branchIndex].ParentControlled = true
}
// Copy requires otherwise it keeps the changes
// Marshal -> unmarshal to create a new object and not keep reference of child objects
marshalled, err := json.Marshal(workflow)
if err != nil {
log.Printf("[ERROR] Failed marshalling parent workflow %s (%s): %s", workflow.Name, workflow.ID, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Suborg distribution failed in marshal"}`))
return
}
newWorkflow := Workflow{}
err = json.Unmarshal(marshalled, &newWorkflow)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling parent workflow %s (%s): %s", workflow.Name, workflow.ID, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Suborg distribution failed in unmarshal"}`))
return
}
// FIXME: Taking the value coming back here
// contains reference objects in the workflow that causes
// e.g. authenticationIds to be reset.
// This is a temporary fix to avoid it.
// FIXME: Removed goroutine. Does it matter?
// Makes the timing problem go away.
diffWorkflowWrapper(newWorkflow)
}
workflow.UpdatedBy = user.Username
if workflow.Public {
workflow.SuborgDistribution = []string{}
}
// Encrypt git backup info
if !workflow.BackupConfig.TokensEncrypted {
if len(workflow.BackupConfig.UploadRepo) > 0 {
parsedKey := fmt.Sprintf("%s_upload_repo", workflow.OrgId)
encryptedToken, err := HandleKeyEncryption([]byte(workflow.BackupConfig.UploadRepo), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed encrypting token for %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadRepo = string(encryptedToken)
workflow.BackupConfig.TokensEncrypted = true
}
}
if len(workflow.BackupConfig.UploadBranch) > 0 {
parsedKey := fmt.Sprintf("%s_upload_branch", workflow.OrgId)
encryptedToken, err := HandleKeyEncryption([]byte(workflow.BackupConfig.UploadBranch), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed encrypting token for %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadBranch = string(encryptedToken)
workflow.BackupConfig.TokensEncrypted = true
}
}
if len(workflow.BackupConfig.UploadUsername) > 0 {
parsedKey := fmt.Sprintf("%s_upload_username", workflow.OrgId)
encryptedToken, err := HandleKeyEncryption([]byte(workflow.BackupConfig.UploadUsername), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed encrypting token for %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadUsername = string(encryptedToken)
workflow.BackupConfig.TokensEncrypted = true
}
}
if len(workflow.BackupConfig.UploadToken) > 0 {
parsedKey := fmt.Sprintf("%s_upload_token", workflow.OrgId)
encryptedToken, err := HandleKeyEncryption([]byte(workflow.BackupConfig.UploadToken), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed encrypting token for %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadToken = string(encryptedToken)
workflow.BackupConfig.TokensEncrypted = true
}
}
}
err = SetWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed saving workflow to database: %s", err)
if workflow.PreviouslySaved {
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
}
org := &Org{}
if org.Id == "" {
org, err = GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org during wf save of %s (org: %s): %s", workflow.ID, user.ActiveOrg.Id, err)
}
}
// This may cause some issues with random slow loads with cross & suborgs, but that's fine (for now)
// FIX: Should only happen for users with this org as the active one
// Org-based workflows may also work
if org.Id != "" {
//for _, loopUser := range org.Users {
// DeleteCache(ctx, fmt.Sprintf("%s_workflows", loopUser.Id))
// DeleteCache(ctx, fmt.Sprintf("%s_workflows", org.Id))
//}
}
if orgUpdated {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed setting org when autoadding apps and updating framework on save workflow save (%s): %s", workflow.ID, err)
} else {
log.Printf("[DEBUG] Successfully updated org %s during save of %s for user %s (%s", user.ActiveOrg.Id, workflow.ID, user.Username, user.Id)
}
}
// Save a backup version of the workflow
// This is to be used for loading in workflows in the future
// It automatically changes the ID to be unique
workflow.OrgId = user.ActiveOrg.Id
workflow.ExecutingOrg = OrgMini{
Id: user.ActiveOrg.Id,
Name: user.ActiveOrg.Name,
}
go SetWorkflowRevision(ctx, workflow)
go func() {
ctx = context.Background()
err = SetGitWorkflow(ctx, workflow, org)
if err != nil {
// Make a notification for this
err = CreateOrgNotification(
ctx,
fmt.Sprintf("Failed setting git workflow for %s (%s): %s", workflow.Name, workflow.ID, err),
fmt.Sprintf("User %s (%s) tried to upload %s (%s) but failed: %s. Make sure there is already a file in the repository, like README.md", user.Username, user.Id, workflow.Name, workflow.ID, err),
fmt.Sprintf("/workflows/%s", workflow.ID),
user.ActiveOrg.Id,
true,
"MEDIUM",
"git",
)
if err != nil {
log.Printf("[WARNING] Failed creating notification for failed git workflow for %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
log.Printf("[WARNING] Failed setting git workflow for %s (%s). Notification created. %s", workflow.Name, workflow.ID, err)
}
}
}()
type returnData struct {
Success bool `json:"success"`
Errors []string `json:"errors"`
}
returndata := returnData{
Success: true,
Errors: workflow.Errors,
}
if !strings.Contains(strings.ToLower(workflow.Name), "ops dashboard") {
log.Printf("[INFO] Saved new version of workflow '%s' (%s) for org %s. User: %s (%s). Actions: %d, Triggers: %d", workflow.Name, fileId, workflow.OrgId, user.Username, user.Id, len(workflow.Actions), len(workflow.Triggers))
}
resp.WriteHeader(200)
newBody, err := json.Marshal(returndata)
if err != nil {
resp.Write([]byte(`{"success": true}`))
return
}
resp.Write(newBody)
}
func HandleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories {
if action.Category == "" {
appName := action.AppName
for _, app := range workflowapps {
if appName != strings.ToLower(app.Name) {
continue
}
if len(app.Categories) > 0 {
log.Printf("[INFO] Setting category for %s: %s", app.Name, app.Categories)
action.Category = app.Categories[0]
break
}
}
//log.Printf("Should find app's categories as it's empty during save")
return categories
}
//log.Printf("Action: %s, category: %s", action.AppName, action.Category)
// FIXME: Make this an "autodiscover" that's controlled by the category itself
// Should just be a list that's looped against :)
newCategory := strings.ToLower(action.Category)
if strings.Contains(newCategory, "case") || strings.Contains(newCategory, "ticket") || strings.Contains(newCategory, "alert") || strings.Contains(newCategory, "mssp") {
categories.Cases.Count += 1
} else if strings.Contains(newCategory, "siem") || strings.Contains(newCategory, "event") || strings.Contains(newCategory, "log") || strings.Contains(newCategory, "search") {
categories.SIEM.Count += 1
} else if strings.Contains(newCategory, "sms") || strings.Contains(newCategory, "comm") || strings.Contains(newCategory, "phone") || strings.Contains(newCategory, "call") || strings.Contains(newCategory, "chat") || strings.Contains(newCategory, "mail") || strings.Contains(newCategory, "phish") {
categories.Communication.Count += 1
} else if strings.Contains(newCategory, "intel") || strings.Contains(newCategory, "crim") || strings.Contains(newCategory, "ti") {
categories.Intel.Count += 1
} else if strings.Contains(newCategory, "sand") || strings.Contains(newCategory, "virus") || strings.Contains(newCategory, "malware") || strings.Contains(newCategory, "scan") || strings.Contains(newCategory, "edr") || strings.Contains(newCategory, "endpoint detection") {
// Sandbox lol
categories.EDR.Count += 1
} else if strings.Contains(newCategory, "vuln") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "fim") || strings.Contains(newCategory, "integrity") {
categories.Assets.Count += 1
} else if strings.Contains(newCategory, "network") || strings.Contains(newCategory, "firewall") || strings.Contains(newCategory, "waf") || strings.Contains(newCategory, "switch") {
categories.Network.Count += 1
} else {
categories.Other.Count += 1
}
return categories
}
// Adds app auth tracking
func UpdateAppAuth(ctx context.Context, auth AppAuthenticationStorage, workflowId, nodeId string, add bool) error {
workflowFound := false
workflowIndex := 0
nodeFound := false
for index, workflow := range auth.Usage {
if workflow.WorkflowId == workflowId {
// Check if node exists
workflowFound = true
workflowIndex = index
for _, actionId := range workflow.Nodes {
if actionId == nodeId {
nodeFound = true
break
}
}
break
}
}
// FIXME: Add a way to use !add to remove
updateAuth := false
if !workflowFound && add {
//log.Printf("[INFO] Adding workflow things to auth!")
usageItem := AuthenticationUsage{
WorkflowId: workflowId,
Nodes: []string{nodeId},
}
auth.Usage = append(auth.Usage, usageItem)
auth.WorkflowCount += 1
auth.NodeCount += 1
updateAuth = true
} else if !nodeFound && add {
//log.Printf("[INFO] Adding node things to auth!")
auth.Usage[workflowIndex].Nodes = append(auth.Usage[workflowIndex].Nodes, nodeId)
auth.NodeCount += 1
updateAuth = true
}
if updateAuth {
//log.Printf("[INFO] Updating auth!")
err := SetWorkflowAppAuthDatastore(ctx, auth, auth.Id)
if err != nil {
log.Printf("[WARNING] Failed UPDATING app auth %s: %s", auth.Id, err)
return err
}
}
return nil
}
func HandleApiGeneration(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting API GEN request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
userInfo, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in apigen: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//log.Printf("IN APIKEY GENERATION")
ctx := GetContext(request)
if request.Method == "GET" {
newUserInfo, err := GenerateApikey(ctx, userInfo)
if err != nil {
log.Printf("[WARNING] Failed to generate apikey for user %s: %s", userInfo.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": ""}`))
return
}
userInfo = newUserInfo
log.Printf("[INFO] Updated apikey for user %s", userInfo.Username)
} else if request.Method == "POST" {
if request.Body == nil {
resp.WriteHeader(http.StatusBadRequest)
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field: user_id"}`)))
return
}
type userId struct {
UserId string `json:"user_id"`
}
var t userId
err = json.Unmarshal(body, &t)
if err != nil {
log.Printf("Failed unmarshaling userId: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unmarshaling. Missing field: user_id"}`)))
return
}
log.Printf("[INFO] Handling post for APIKEY gen FROM user %s. Userchange: %s!", userInfo.Username, t.UserId)
if userInfo.Role != "admin" {
log.Printf("[AUDIT] %s tried and failed to change apikey for %s (2)", userInfo.Username, t.UserId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You need to be admin to change others' apikey"}`)))
return
}
foundUser, err := GetUser(ctx, t.UserId)
if err != nil {
log.Printf("[INFO] Can't find user %s (apikey gen): %s", t.UserId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
// FIXME: May not be good due to different roles in different organizations.
if foundUser.Role == "admin" {
log.Printf("[AUDIT] %s tried and failed to change apikey for %s. Skipping because users' role is admin", userInfo.Username, t.UserId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change the apikey of another admin"}`)))
return
}
newUserInfo, err := GenerateApikey(ctx, *foundUser)
if err != nil {
log.Printf("Failed to generate apikey for user %s: %s", foundUser.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
foundUser = &newUserInfo
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, foundUser.Username, foundUser.Verified, foundUser.ApiKey)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "username": "%s", "verified": %t, "apikey": "%s"}`, userInfo.Username, userInfo.Verified, userInfo.ApiKey)))
}
func HandleSettings(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Handle Settings request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
userInfo, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in settings: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newObject := SettingsReturn{
Success: true,
Username: userInfo.Username,
Verified: userInfo.Verified,
Apikey: userInfo.ApiKey,
Image: userInfo.PublicProfile.GithubAvatar,
}
newjson, err := json.Marshal(newObject)
if err != nil {
log.Printf("[ERROR] Failed unmarshal in get settings: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed handling your user"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func CleanCreds(user *User, currentUser User) *User {
user.Password = ""
user.ApiKey = ""
user.Session = ""
user.UsersLastSession = ""
user.VerificationToken = ""
user.ValidatedSessionOrgs = []string{}
if currentUser.SupportAccess {
return user
}
//user.Orgs = []string{}
handledOrgs := []string{}
// Quick deduplication.
for _, org := range user.Orgs {
if ArrayContains(handledOrgs, org) {
continue
}
handledOrgs = append(handledOrgs, org)
}
user.Orgs = handledOrgs
user.Authentication = []UserAuth{}
user.PrivateApps = []WorkflowApp{}
// let's come back to this
user.MFA = MFAInfo{
Active: user.MFA.Active,
}
user.ActiveOrg = OrgMini{}
if !user.SupportAccess {
user.LoginInfo = []LoginInfo{}
}
user.LoginInfo = []LoginInfo{}
// user.LoginType = "DELETED"
// login type options is either: SSO, OPENID or empty.
// we add deleted in other cleanups.
// there's some frontend logic supporting this at weird places
// let's just cleanup LoginType here for now.
if user.LoginType != "DELETED" {
user.LoginType = ""
}
//user.Role = "user"
return user
}
func HandleGetUsers(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Users request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get users: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin (%s) and can't list users.", user.Role)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org in get users: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org when listing users"}`))
return
}
newUsers := []User{}
for _, item := range org.Users {
if len(item.Username) == 0 {
continue
}
// Get the actual user
foundUser, err := GetUser(ctx, item.Id)
if err != nil {
log.Printf("[WARNING] Failed getting user in get users: %s", err)
} else {
// Overrides to ensure the user we are returning
// is accurate and not an org copy. Keeping roles from
// org, as that controls the actual roles.
newItem := *foundUser
newItem.Role = item.Role
newItem.Roles = []string{item.Role}
newItem.ActiveOrg = item.ActiveOrg
item = newItem
}
if item.Id != user.Id {
item.ApiKey = ""
}
item.ApiKey = ""
item.Password = ""
item.Session = ""
item.UsersLastSession = ""
item.VerificationToken = ""
item.ValidatedSessionOrgs = []string{}
item.Orgs = []string{}
item.Authentication = []UserAuth{}
item.PrivateApps = []WorkflowApp{}
item.MFA = MFAInfo{
Active: item.MFA.Active,
}
item.ActiveOrg = OrgMini{}
if !user.SupportAccess {
item.LoginInfo = []LoginInfo{}
}
// Will get from cache 2nd time so this is fine.
if user.Id == item.Id {
item.Orgs = user.Orgs
item.Active = user.Active
//item.MFA = user.MFA
} else {
// Only add IF the admin querying it has access, meaning only show what you yourself have access toMFAInfo
allOrgs := []string{}
handledOrgs := []string{}
for _, orgname := range foundUser.Orgs {
found := false
if ArrayContains(handledOrgs, orgname) {
continue
}
handledOrgs = append(handledOrgs, orgname)
for _, userOrg := range user.Orgs {
if userOrg == orgname {
found = true
break
}
}
if found {
allOrgs = append(allOrgs, orgname)
}
}
//log.Printf("[DEBUG] Added %d org(s) for user %s (%s) - get users", len(allOrgs), foundUser.Username, foundUser.Id)
//item.MFA = foundUser.MFA
item.Verified = foundUser.Verified
item.Active = foundUser.Active
item.Orgs = allOrgs
}
if len(item.Orgs) == 0 {
item.Orgs = append(item.Orgs, user.ActiveOrg.Id)
}
if user.SupportAccess {
item.Orgs = foundUser.Orgs
}
newUsers = append(newUsers, item)
}
if project.Environment == "cloud" {
orgUsers, err := GetUsersByOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org users for support access: %s", err)
} else {
for _, orgUser := range orgUsers {
// orgUser = *CleanCreds(&orgUser)
found := false
for _, existingUser := range newUsers {
if existingUser.Id == orgUser.Id {
found = true
break
}
}
if found {
continue
}
orgUser.ApiKey = ""
orgUser.Password = ""
orgUser.Session = ""
orgUser.UsersLastSession = ""
orgUser.VerificationToken = ""
orgUser.ValidatedSessionOrgs = []string{}
//orgUser.Orgs = []string{}
orgUser.Authentication = []UserAuth{}
orgUser.PrivateApps = []WorkflowApp{}
orgUser.MFA = MFAInfo{
Active: orgUser.MFA.Active,
}
orgUser.ActiveOrg = OrgMini{}
if !orgUser.SupportAccess {
orgUser.LoginInfo = []LoginInfo{}
}
//orgUser.Deleted = true
orgUser.LoginType = "DELETED"
orgUser.Role = "user"
// Will get from cache 2nd time so this is fine.
if user.Id == orgUser.Id {
orgUser.Orgs = user.Orgs
orgUser.Active = user.Active
//item.MFA = user.MFA
} else {
// Only add IF the admin querying it has access, meaning only show what you yourself have access toMFAInfo
foundUser, err := GetUser(ctx, orgUser.Id)
if err == nil {
allOrgs := []string{}
handledOrgs := []string{}
for _, orgname := range foundUser.Orgs {
found := false
if ArrayContains(handledOrgs, orgname) {
continue
}
handledOrgs = append(handledOrgs, orgname)
for _, userOrg := range user.Orgs {
if userOrg == orgname {
found = true
break
}
}
if found {
allOrgs = append(allOrgs, orgname)
}
}
//log.Printf("[DEBUG] Added %d org(s) for user %s (%s) - get users", len(allOrgs), foundUser.Username, foundUser.Id)
//item.MFA = foundUser.MFA
orgUser.Verified = foundUser.Verified
orgUser.Active = foundUser.Active
orgUser.Orgs = allOrgs
}
if user.SupportAccess {
orgUser.Orgs = foundUser.Orgs
}
}
if len(orgUser.Orgs) == 0 {
orgUser.Orgs = append(orgUser.Orgs, user.ActiveOrg.Id)
}
newUsers = append(newUsers, orgUser)
}
}
}
deduplicatedUsers := []User{}
for _, item := range newUsers {
found := false
for _, tmpUser := range deduplicatedUsers {
if tmpUser.Username == item.Username {
found = true
break
}
}
if !found {
cleanedUser := CleanCreds(&item, user)
deduplicatedUsers = append(deduplicatedUsers, *cleanedUser)
}
}
newjson, err := json.Marshal(deduplicatedUsers)
if err != nil {
log.Printf("[WARNING] Failed unmarshal in getusers: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
// Partners controllers
func HandleGetAllPartners(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Partner request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
ctx := GetContext(request)
partners, err := GetAllPartners(ctx)
if err != nil {
log.Printf("[ERROR] Failed to get partners: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to get partners"}`))
return
}
// Filter partners to only include public ones
var publicPartners []Partner
for _, partner := range partners {
if partner.Public {
publicPartners = append(publicPartners, partner)
}
}
if len(publicPartners) == 0 {
log.Printf("[DEBUG] No public partners found")
resp.WriteHeader(http.StatusNotFound)
resp.Write([]byte(`{"success": false, "reason": "No public partners found"}`))
return
}
type returnStruct struct {
Success bool `json:"success"`
Partners []Partner `json:"data"`
}
allPartners := returnStruct{
Success: true,
Partners: partners,
}
response, err := json.Marshal(allPartners)
if err != nil {
log.Printf("[ERROR] Failed to marshal partners: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to process partners data"}`))
return
}
resp.WriteHeader(http.StatusOK)
resp.Write(response)
}
func HandleGetPartner(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Partner request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in getting partner: %s. Continuing because it may be visible to it's owner", userErr)
}
var partnerId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) > 4 {
partnerId = location[4]
}
}
if len(partnerId) == 0 {
log.Printf("[ERROR] Partner ID is missing in request: %s", request.URL.String())
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "Missing partner ID"}`))
return
}
ctx := GetContext(request)
partner, err := GetPartnerById(ctx, partnerId)
if err != nil {
log.Printf("[ERROR] Failed to get partner: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to get partner"}`))
return
}
if len(partner.Id) == 0 {
log.Printf("[ERROR] Partner ID is empty for partner: %v", partner)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "Partner ID is empty"}`))
return
}
if !partner.Public {
if partner.Id != user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s (%s) tried to access non-public partner %s (%s)", user.Username, user.Id, partner.Name, partner.Id)
resp.WriteHeader(http.StatusForbidden)
resp.Write([]byte(`{"success": false, "reason": "This partner is not public"}`))
return
}
}
type returnStruct struct {
Success bool `json:"success"`
Partner *Partner `json:"partner"`
}
partnerData := returnStruct{
Success: true,
Partner: partner,
}
response, err := json.Marshal(partnerData)
if err != nil {
log.Printf("[ERROR] Failed to marshal partner: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to process partner data"}`))
return
}
resp.WriteHeader(http.StatusOK)
resp.Write(response)
}
func HandlePasswordChange(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Password Change request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
if request.Body == nil {
resp.WriteHeader(http.StatusBadRequest)
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
// Get the current user - check if they're admin or the "username" user.
var t PasswordChange
err = json.Unmarshal(body, &t)
if err != nil {
log.Printf("Failed unmarshaling")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
userInfo, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in password change: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[AUDIT] Handling password change for %s from %s (%s)", t.Username, userInfo.Username, userInfo.Id)
curUserFound := false
if t.Username != userInfo.Username {
log.Printf("[WARNING] Bad username during password change for %s.", t.Username)
if project.Environment == "cloud" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Not allowed to change others' passwords in cloud"}`))
return
}
} else if t.Username == userInfo.Username {
curUserFound = true
}
// Checking current user changing another user
if userInfo.Role != "admin" {
if t.Newpassword != t.Newpassword2 {
err := "Passwords don't match"
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
if project.Environment == "cloud" {
if len(t.Newpassword) < 10 || len(t.Newpassword2) < 10 {
err := "Passwords too short - 2"
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
}
}
// Current password
err = CheckPasswordStrength("", t.Newpassword)
if err != nil {
log.Printf("[INFO] Bad password strength: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
ctx := GetContext(request)
foundUser := User{}
if !curUserFound {
users, err := FindUser(ctx, strings.ToLower(strings.TrimSpace(t.Username)))
if err != nil && len(users) == 0 {
log.Printf("[WARNING] Failed getting user %s: %s", t.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
if len(users) != 1 {
log.Printf(`[WARNING] Found multiple or no users with the same username: %s: %d`, t.Username, len(users))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), t.Username)))
return
}
foundUser = users[0]
orgFound := false
if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id {
orgFound = true
} else {
for _, item := range foundUser.Orgs {
if item == userInfo.ActiveOrg.Id {
orgFound = true
break
}
}
}
if !orgFound {
log.Printf("[AUDIT] User %s (%s) is admin, but can't change user's (%s) password outside their own org.", userInfo.Username, userInfo.Id, foundUser.Username)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org (2)."}`)))
return
}
} else {
// Admins can re-generate others' passwords as well (onprem only).
err = bcrypt.CompareHashAndPassword([]byte(userInfo.Password), []byte(t.Currentpassword))
if err != nil {
log.Printf("[WARNING] Bad password for %s: %s", userInfo.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
foundUser = userInfo
}
if len(foundUser.Id) == 0 {
log.Printf("[WARNING] Something went wrong in password reset: couldn't find user.")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Newpassword), 8)
if err != nil {
log.Printf("[ERROR] New password failure for %s: %s", userInfo.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
foundUser.Password = string(hashedPassword)
cacheKey := fmt.Sprintf("session_%s", foundUser.Session)
DeleteCache(ctx, cacheKey)
foundUser.Session = ""
err = SetUser(ctx, &foundUser, true)
if err != nil {
log.Printf("[ERROR] Problem fixing password for user %s: %s", userInfo.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Session invalidated. You need to re-login"}`)))
}
// Can check against HIBP etc?
// Removed for localhost
func CheckPasswordStrength(username, password string) error {
// Check password strength here
if project.Environment == "cloud" {
if len(password) < 10 {
return errors.New("Minimum password length is 10.")
}
if len(password) > 128 {
return errors.New("Maximum password length is 128.")
}
if username == password {
return errors.New("Username and password can't be the same.")
}
} else {
// Onprem~
if len(password) < 10 {
return errors.New("Minimum password length is 10.")
}
}
//if len(password) > 128 {
// return errors.New("Maximum password length is 128.")
//}
//re := regexp.MustCompile("[0-9]+")
//if len(re.FindAllString(password, -1)) == 0 {
// return errors.New("Password must contain a number")
//}
//re = regexp.MustCompile("[a-z]+")
//if len(re.FindAllString(password, -1)) == 0 {
// return errors.New("Password must contain a lower case char")
//}
//re = regexp.MustCompile("[A-Z]+")
//if len(re.FindAllString(password, -1)) == 0 {
// return errors.New("Password must contain an upper case char")
//}
return nil
}
func SendHookResult(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in send hook results: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
_ = user
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 32 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
ctx := GetContext(request)
hook, err := GetHook(ctx, workflowId)
if err != nil {
log.Printf("[WARNING] Failed getting hook %s (send): %s", workflowId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Body data error: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("SET the hook results for %s to %s", workflowId, body)
// FIXME - set the hook result in the DB somehow as interface{}
// FIXME - should the hook do the transform? Hmm
b, err := json.Marshal(hook)
if err != nil {
log.Printf("Failed marshalling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(b))
return
}
func HandleGetHook(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get hook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var workflowId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
workflowId = location[4]
}
if len(workflowId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "message": "ID not valid"}`))
return
}
ctx := GetContext(request)
hook, err := GetHook(ctx, workflowId)
if err != nil {
log.Printf("[WARNING] Failed getting hook %s (get hook): %s", workflowId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != hook.Owner && user.Role != "scheduler" {
log.Printf("Wrong user (%s) for hook %s", user.Username, hook.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
b, err := json.Marshal(hook)
if err != nil {
log.Printf("Failed marshalling: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(b))
return
}
func DuplicateWorkflow(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Removed check here as it may be a public workflow
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in duplicate workflow: %s. Continuing because it may be public IF cloud.", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("\n\n[WARNING] Workflow ID when duplicating workflow is not valid: %s. URL: %s", fileId, request.URL.String())
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when duplicating workflow is not valid"}`))
return
}
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Workflow %s doesn't exist.", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed finding workflow"}`))
return
}
// Check workflow.Sharing == private / public / org too
isOwner := false
if user.Id != workflow.Owner || len(user.Id) == 0 {
// Added org-reader as the user should be able to read everything in an org
//if workflow.OrgId == user.ActiveOrg.Id && (user.Role == "admin" || user.Role == "org-reader") {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as the org is same (duplicate workflow)", user.Username, workflow.ID)
isOwner = true
} else if workflow.Public {
log.Printf("[AUDIT] Letting user %s access workflow %s because it's public (duplicate workflow)", user.Username, workflow.ID)
// Only for Read-Only. No executions or impersonations.
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow %s (duplicate workflow)", user.Username, workflow.ID)
isOwner = true
} else {
log.Printf("[AUDIT] Wrong user %s (%s) for workflow '%s' (duplicate workflow). Verified: %t, Active: %t, SupportAccess: %t, Username: %s", user.Username, user.Id, workflow.ID, user.Verified, user.Active, user.SupportAccess, user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if len(workflow.ParentWorkflowId) > 0 {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Can't duplicate a workflow distributed from a parent org"}`))
return
}
newId := uuid.NewV4().String()
log.Printf("[DEBUG] Duplicated workflow %s for user %s with new ID %s", workflow.ID, user.Username, newId)
type WorkflowDupe struct {
Name string `json:"name"`
}
var t WorkflowDupe
err = json.NewDecoder(request.Body).Decode(&t)
if err != nil {
log.Printf("[WARNING] Failed decoding workflow dupe: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
if len(t.Name) == 0 {
t.Name = workflow.Name + " (copy)"
}
workflow.Name = t.Name
workflow.Owner = user.Id
workflow.ID = newId
workflow.OrgId = user.ActiveOrg.Id
workflow.Org = []OrgMini{user.ActiveOrg}
workflow.ExecutingOrg = user.ActiveOrg
workflow.Created = 0
workflow.Edited = 0
err = SetWorkflow(ctx, *workflow, newId)
if err != nil {
log.Printf("[WARNING] Failed setting workflow %s: %s", workflow.ID, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
_ = isOwner
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, workflow.ID)))
return
}
func GenerateWorkflowFromParent(ctx context.Context, workflow Workflow, parentOrgId, subOrgId string) (*Workflow, error) {
// FIXME: This check should NOT exist as it could cause issues
// with same names in different orgs.
childOrgWorkflows, err := GetAllWorkflowsByQuery(ctx, User{
Role: "admin",
ActiveOrg: OrgMini{
Id: subOrgId,
},
}, 250, "")
if err != nil {
log.Printf("[ERROR] Failed getting workflows for suborg %s: %s", subOrgId, err)
} else {
for _, foundWorkflow := range childOrgWorkflows {
if foundWorkflow.Name == workflow.Name {
log.Printf("[ERROR] Found a workflow with the same ID (%s) in suborg %s (%s).", foundWorkflow.ID, subOrgId, foundWorkflow.Name)
return &foundWorkflow, nil
}
}
}
DeleteCache(ctx, fmt.Sprintf("%s_workflows", workflow.OrgId))
DeleteCache(ctx, fmt.Sprintf("%s_workflows", subOrgId))
DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", workflow.ID))
parentWorkflowId := workflow.ID
// Make a copy of the workflow, and set parent/child relationships
// Seed random based on the existing workflow + suborg to make sure we only make one
seedString := fmt.Sprintf("%s_%s", workflow.ID, subOrgId)
hash := sha1.New()
hash.Write([]byte(seedString))
hashBytes := hash.Sum(nil)
uuidBytes := make([]byte, 16)
copy(uuidBytes, hashBytes)
newId := uuid.Must(uuid.FromBytes(uuidBytes)).String()
DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", newId))
// before doing anything, verify if the parent workflow is a child workflow itself
if len(workflow.ParentWorkflowId) > 0 && len(workflow.SuborgDistribution) > 0 {
log.Printf("[ERROR] Disabled suborg distribution for child workflow %s (%s). This usually only happens due to an ID bug somewhere from parent org (%s) to child org (%s)", workflow.ID, workflow.Name, parentOrgId, subOrgId)
workflow.Errors = append(workflow.Errors, fmt.Sprintf("Suborg distribution disabled automatically in child workflow %s.", workflow.Name))
workflow.SuborgDistribution = []string{}
err = SetWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed setting workflow %s while overwriting during SubOrgDistribution error: %s", workflow.ID, err)
return nil, err
}
return &Workflow{}, errors.New("Parent workflow is a child workflow itself")
}
// Returns the existing one in case it has been made in the past
// This is to ensure old nodes still exist.
foundWorkflow, err := GetWorkflow(ctx, newId)
if err == nil && foundWorkflow.ID == newId && foundWorkflow.ParentWorkflowId == parentWorkflowId {
log.Printf("[INFO] Found existing child workflow %s for %s", newId, parentWorkflowId)
return foundWorkflow, nil
}
if !ArrayContains(workflow.ChildWorkflowIds, newId) {
log.Printf("[INFO] Adding new child workflow %s to %s", newId, parentWorkflowId)
workflow.ChildWorkflowIds = append(workflow.ChildWorkflowIds, newId)
DeleteCache(ctx, fmt.Sprintf("%s_workflows", workflow.OrgId))
err = SetWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed adding new child workflow %s: %s", newId, err)
} else {
log.Printf("[AUDIT] Added new child workflow of %s in suborg %s", workflow.ID, subOrgId)
}
}
newWf := workflow
newWf.ID = newId
newWf.SuborgDistribution = []string{}
newWf.ChildWorkflowIds = []string{}
newWf.ParentWorkflowId = parentWorkflowId
newWf.Org = []OrgMini{
OrgMini{
Id: subOrgId,
},
}
newWf.OrgId = subOrgId
newWf.ExecutingOrg = OrgMini{
Id: subOrgId,
}
newWf.Created = 0
newWf.Edited = 0
defaultEnvironment := "cloud"
for _, action := range newWf.Actions {
if len(action.Environment) > 0 {
defaultEnvironment = action.Environment
break
}
}
envs, err := GetEnvironments(ctx, subOrgId)
for _, env := range envs {
if env.Default {
defaultEnvironment = env.Name
break
}
}
// Letting full replication occur
for actionIndex, _ := range newWf.Actions {
//workflow.Actions[actionIndex].ParentControlled = true
//workflow.Actions[actionIndex].Environment = defaultEnvironment
newWf.Actions[actionIndex].ParentControlled = true
newWf.Actions[actionIndex].Environment = defaultEnvironment
}
// Triggers are handled in the diff instead.
newWf.Triggers = []Trigger{}
//log.Printf("[INFO] Generated child workflow %s (%s) for %s (%s)", childWorkflow.Name, childWorkflow.ID, parentWorkflow.Name, parentWorkflow.ID)
// FIXME: Send a save request instead? That way
// propagation can keep going down.
// TODO: Not implemented due to recursion issues.
err = SetWorkflow(ctx, newWf, newWf.ID)
if err != nil {
log.Printf("[DEBUG] Failed setting new child workflow of ID %s (%s): %s", workflow.ID, newWf.ID, err)
}
// Diffs them & makes changes in the child directly
diffWorkflows(newWf, workflow, true)
return &newWf, err
}
func GetSpecificWorkflow(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Removed check here as it may be a public workflow
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in getting specific workflow: %s. Continuing because it may be public IF cloud.", err)
/*
// No need to keep workflow forms to cloud only. Public access available from February 2025.
if project.Environment != "cloud" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
*/
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
log.Printf("[WARNING] Workflow ID when getting workflow is not valid: %s. URL: %s", fileId, request.URL.String())
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`))
return
}
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId)
if err != nil || len(workflow.ID) == 0 {
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting NOT FOUND workflow request for %s to main site handler (shuffler.io)", fileId)
RedirectUserRequest(resp, request)
return
}
}
log.Printf("[WARNING] Workflow %s doesn't exist.", fileId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed finding workflow"}`))
return
}
// Special case to handle suborg distribution workflow loading
if len(workflow.SuborgDistribution) > 0 {
for _, orgId := range workflow.SuborgDistribution {
if orgId != user.ActiveOrg.Id {
continue
}
log.Printf("[AUDIT] User %s is accessing workflow %s from a suborg that has access (get workflow)", user.Username, workflow.ID)
// Check local workflows to see if a local version of the workflow exists. Should NOT be able to see the parents' workflow directly (?)
workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "")
if err != nil {
log.Printf("[WARNING] Failed getting workflows in get workflow with suborg distrib. Auth should fail.: %s", err)
} else {
found := false
for _, childWorkflow := range workflows {
if childWorkflow.ParentWorkflowId != workflow.ID {
continue
}
found = true
fileId = childWorkflow.ID
workflow = &childWorkflow
break
}
if !found {
log.Printf("[AUDIT] Failed to find existing workflow for user %s in suborg %s. Making replica.", user.Username, orgId)
parentWorkflowOrgId := workflow.OrgId
childOrgId := orgId
newWf, err := GenerateWorkflowFromParent(ctx, *workflow, parentWorkflowOrgId, childOrgId)
if err != nil {
log.Printf("[ERROR] Failed setting new child workflow %s: %s", newWf.ID, err)
} else {
log.Printf("[AUDIT] Created new child workflow of %s for user %s in suborg %s", workflow.ID, user.Username, orgId)
workflow = newWf
}
DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.ActiveOrg.Id))
} else {
log.Printf("[AUDIT] Found existing workflow for user %s in suborg %s. Loading.", user.Username, orgId)
}
}
break
}
}
// Check the URL source path to include /form or /run
isOwner := false
if user.Id != workflow.Owner || len(user.Id) == 0 {
// Added org-reader as the user should be able to read everything in an org
//if workflow.OrgId == user.ActiveOrg.Id && (user.Role == "admin" || user.Role == "org-reader") {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as the org is same (get workflow)", user.Username, workflow.ID)
isOwner = true
} else if workflow.Public {
log.Printf("[AUDIT] Letting user %s access workflow %s because it's public", user.Username, workflow.ID)
// Only for Read-Only. No executions or impersonations.
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow %s (get workflow)", user.Username, workflow.ID)
isOwner = true
} else if workflow.Sharing == "form" {
log.Printf("[AUDIT] Letting user %s access workflow %s because it's a form. Sanitized format.", user.Username, workflow.ID)
// Execute-Only. No executions or impersonations.
// Remaking the workflow intirely to ONLY include relevant stuff, and be future-proof
//user.ActiveOrg.Id = workflow.OrgId
workflow = &Workflow{
Name: workflow.Name,
ID: workflow.ID,
Owner: workflow.Owner,
OrgId: workflow.OrgId,
FormControl: workflow.FormControl,
Sharing: workflow.Sharing,
Description: workflow.Description,
InputQuestions: workflow.InputQuestions,
}
} else {
log.Printf("[AUDIT] Wrong user %s (%s) for workflow '%s' (get workflow). Verified: %t, Active: %t, SupportAccess: %t, Username: %s", user.Username, user.Id, workflow.ID, user.Verified, user.Active, user.SupportAccess, user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if len(workflow.Actions) == 0 {
workflow.Actions = []Action{}
}
if len(workflow.Branches) == 0 {
workflow.Branches = []Branch{}
}
if len(workflow.Triggers) == 0 {
workflow.Triggers = []Trigger{}
}
if len(workflow.Errors) == 0 {
workflow.Errors = []string{}
}
for key, _ := range workflow.Actions {
// RefUrl not necessary anymore, as we migrated to getting apps during exec
workflow.Actions[key].ReferenceUrl = ""
// Never helpful when this is red
if workflow.Actions[key].AppName == "Shuffle Tools" {
workflow.Actions[key].IsValid = true
}
if !workflow.Actions[key].IsValid {
//log.Printf("[AUDIT] Invalid action in workflow '%s' (%s): '%s' (%s)", workflow.Name, workflow.ID, workflow.Actions[key].Label, workflow.Actions[key].ID)
// Check if all fields are set
// Check if auth is set (autofilled)
isValid := true
for _, param := range workflow.Actions[key].Parameters {
if param.Required && len(param.Value) == 0 {
isValid = false
break
}
}
if isValid {
workflow.Actions[key].IsValid = true
}
}
}
// Getting in here during schemaless is normal
if len(workflow.Name) == 0 && len(workflow.ID) == 0 {
//log.Printf("[ERROR] Workflow has no name or ID, hence may not exist. Reference ID (maybe from Algolia?: %s)", fileId)
// FIXME: Cloud + redirects? Can we find copies of workflows to redirect to?
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting NOT FOUND workflow request for %s to main site handler (shuffler.io) (2)", fileId)
RedirectUserRequest(resp, request)
return
}
}
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No workflow found"}`))
return
}
// Never load without a node
if len(workflow.Actions) == 0 && workflow.Sharing != "form" {
// Append
nodeId := uuid.NewV4().String()
workflow.Start = nodeId
envName := "cloud"
if project.Environment != "cloud" {
envName = "Shuffle"
}
workflowapps, err := GetPrioritizedApps(ctx, user)
if err == nil {
for _, item := range workflowapps {
//log.Printf("NAME: %s", item.Name)
if (item.Name == "Shuffle Tools" || item.Name == "Shuffle-Tools") && item.AppVersion == "1.2.0" {
newAction := Action{
Label: "Change Me",
Name: "repeat_back_to_me",
Environment: envName,
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "call",
Value: "Hello world",
Example: "Repeating: Hello World",
Multiline: true,
},
},
Priority: 0,
Errors: []string{},
ID: nodeId,
IsValid: true,
IsStartNode: true,
Sharing: true,
PrivateID: "",
SmallImage: "",
AppName: "Shuffle Tools",
AppVersion: "1.2.0",
AppID: item.ID,
LargeImage: item.LargeImage,
}
newAction.Position = Position{
X: 449.5,
Y: 446.1,
}
workflow.Actions = append(workflow.Actions, newAction)
break
}
}
}
}
workflowapps := []WorkflowApp{}
if len(user.Id) > 0 && len(user.ActiveOrg.Id) > 0 {
workflowapps, err = GetPrioritizedApps(ctx, user)
if err != nil {
log.Printf("[WARNING] Error: Failed getting workflowapps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// Handle app versions & upgrades
for _, action := range workflow.Actions {
actionApp := strings.ToLower(strings.Replace(action.AppName, " ", "", -1))
for _, app := range workflowapps {
if strings.ToLower(strings.Replace(app.Name, " ", "", -1)) != actionApp {
continue
}
if len(app.Versions) <= 1 {
continue
}
v2, err := semver.NewVersion(action.AppVersion)
if err != nil {
log.Printf("[ERROR] Failed parsing original app version %s: %s", app.AppVersion, err)
continue
}
newVersion := ""
for _, loopedApp := range app.Versions {
if action.AppVersion == loopedApp.Version {
continue
}
appConstraint := fmt.Sprintf("< %s", loopedApp.Version)
c, err := semver.NewConstraint(appConstraint)
if err != nil {
log.Printf("[ERROR] Failed preparing constraint %s: %s", appConstraint, err)
continue
}
if c.Check(v2) {
newVersion = loopedApp.Version
action.AppVersion = loopedApp.Version
}
}
if len(newVersion) > 0 {
newError := fmt.Sprintf("App %s has version %s available.", app.Name, newVersion)
if !ArrayContains(workflow.Errors, newError) {
workflow.Errors = append(workflow.Errors, newError)
}
}
}
}
if workflow.Public {
workflow.BackupConfig = BackupConfig{}
workflow.ExecutingOrg = OrgMini{}
workflow.Org = []OrgMini{}
isPartner := false
if len(workflow.Owner) > 0 {
// From now on, we will have the owner ID (Org || User)
org, err := GetOrg(ctx, workflow.Owner)
if err == nil {
// So the owner is org not user now check if the org is a partner
if org.LeadInfo.TechPartner || org.LeadInfo.IntegrationPartner || org.LeadInfo.DistributionPartner || org.LeadInfo.ServicePartner || org.LeadInfo.ChannelPartner {
isPartner = true
}
// If the org is a partner, Check if that org owns the workflow (if not then don't show the owner id)
if isPartner && workflow.Owner != user.ActiveOrg.Id && user.Role != "admin" {
workflow.Owner = ""
}
if !isPartner {
workflow.Owner = ""
}
} else {
// So the owner is user not org
log.Printf("[WARNING] Failed getting org for public workflow'%s': %s", workflow.ID, err)
// Check if the active user is the actual owner of that public workflow
if workflow.Owner != user.Id {
// If the user doesn't own the workflow, then don't show the owner
workflow.Owner = ""
}
}
} else {
// Backward compatibility: Previously we did not store owner or orgId when publishing workflows, only the "updated by" field is there which can be used to determine the owner.
if user.Username == workflow.UpdatedBy {
workflow.Owner = user.Id
}
}
workflow.OrgId = ""
if !isOwner {
workflow.PreviouslySaved = false
workflow.ID = ""
}
}
if workflow.BackupConfig.TokensEncrypted {
parsedKey := fmt.Sprintf("%s_upload_token", workflow.OrgId)
newValue, err := HandleKeyDecryption([]byte(workflow.BackupConfig.UploadToken), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting token for workflow %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadToken = string(newValue)
}
parsedKey = fmt.Sprintf("%s_upload_username", workflow.OrgId)
newValue, err = HandleKeyDecryption([]byte(workflow.BackupConfig.UploadUsername), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting username for workflow %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadUsername = string(newValue)
}
parsedKey = fmt.Sprintf("%s_upload_repo", workflow.OrgId)
newValue, err = HandleKeyDecryption([]byte(workflow.BackupConfig.UploadRepo), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting repo for workflow %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadRepo = string(newValue)
}
parsedKey = fmt.Sprintf("%s_upload_branch", workflow.OrgId)
newValue, err = HandleKeyDecryption([]byte(workflow.BackupConfig.UploadBranch), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting branch for org %s (%s): %s", workflow.Name, workflow.ID, err)
} else {
workflow.BackupConfig.UploadBranch = string(newValue)
}
}
//Check if workflow trigger schedule is in sync with the gcp cron job
if project.Environment == "cloud" && workflow.Triggers != nil {
var wg sync.WaitGroup
triggerMutex := sync.Mutex{}
for index, trigger := range workflow.Triggers {
if trigger.TriggerType == "SCHEDULE" {
wg.Add(1)
go func(index int, trigger Trigger) {
defer wg.Done()
// Check if the schedule is in sync with the gcp cron job
GcpSchedule, err := GetGcpSchedule(ctx, trigger.ID)
if err != nil {
log.Printf("[ERROR] Failed getting gcp schedule for trigger %s: %s", trigger.ID, err)
triggerMutex.Lock()
workflow.Triggers[index].Status = "stopped"
triggerMutex.Unlock()
} else {
triggerMutex.Lock()
workflow.Triggers[index].Status = GcpSchedule.Status
triggerMutex.Unlock()
}
}(index, trigger)
}
}
wg.Wait()
//SetWorkflow(ctx, *workflow, workflow.ID)
}
log.Printf("[INFO] Got new version of workflow %s (%s) for org %s and user %s (%s). Actions: %d, Triggers: %d", workflow.Name, workflow.ID, user.ActiveOrg.Id, user.Username, user.Id, len(workflow.Actions), len(workflow.Triggers))
body, err := json.Marshal(workflow)
if err != nil {
log.Printf("[WARNING] Failed workflow GET marshalling: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(body)
}
func DeleteUser(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting User request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
userInfo, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in delete user: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if userInfo.Role != "admin" {
log.Printf("[DEBUG] Wrong user (%s) when deleting - must be admin", userInfo.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Must be admin"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var userId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
userId = location[4]
}
ctx := GetContext(request)
userId, err := url.QueryUnescape(userId)
if err != nil {
log.Printf("[WARNING] Failed decoding user %s: %s", userId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
if userId == userInfo.Id {
log.Printf("[WARNING] Can't change activation of your own user %s (%s)", userInfo.Username, userInfo.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change activation of your own user"}`)))
return
}
foundUser, err := GetUser(ctx, userId)
if err != nil {
log.Printf("[WARNING] Can't find user %s (delete user): %s", userId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
// Overwrite incase the user is in the same org
// This could be a way to jump into someone elses organisation if the user already has the correct org name without correct name.
if foundUser.ActiveOrg.Id == "" && foundUser.ActiveOrg.Name == userInfo.ActiveOrg.Name && len(foundUser.Orgs) == 0 {
foundUser.ActiveOrg.Id = string(userInfo.ActiveOrg.Id)
}
orgFound := false
if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id {
orgFound = true
} else {
for _, item := range foundUser.Orgs {
if item == userInfo.ActiveOrg.Id {
orgFound = true
break
}
}
}
// Edge-case: foundUser.Orgs is empty and foundUser.ActiveOrg is empty,
// but the user exists in the admin's Org.Users list.
// Handle removal self-contained and return early.
if !orgFound {
adminOrg, err := GetOrg(ctx, userInfo.ActiveOrg.Id)
if err == nil {
for i, orgUser := range adminOrg.Users {
if orgUser.Id == foundUser.Id {
orgFound = true
adminOrg.Users = append(adminOrg.Users[:i], adminOrg.Users[i+1:]...)
err = SetOrg(ctx, *adminOrg, adminOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org (delete user %s) %s: %s", foundUser.Username, adminOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed removing user from org"}`))
return
}
log.Printf("[AUDIT] User %s (%s) successfully removed %s from org %s (edge-case: user had no org references)", userInfo.Username, userInfo.Id, foundUser.Username, userInfo.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
return
}
}
}
}
if !orgFound && !userInfo.SupportAccess {
log.Printf("[AUDIT] User %s (%s) is admin, but can't delete users outside their own org.", userInfo.Username, userInfo.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org (1)."}`)))
return
}
// NEW
neworgs := []string{}
for _, orgid := range foundUser.Orgs {
if orgid == userInfo.ActiveOrg.Id {
continue
} else {
// Automatically setting to first one
if foundUser.ActiveOrg.Id == userInfo.ActiveOrg.Id {
foundUser.ActiveOrg.Id = orgid
}
}
neworgs = append(neworgs, orgid)
}
if foundUser.ActiveOrg.Id == userInfo.ActiveOrg.Id {
foundUser.ActiveOrg.Id = ""
foundUser.ActiveOrg.Name = ""
}
foundUser.Orgs = neworgs
if len(foundUser.Orgs) == 0 {
log.Printf("[INFO] User %s (%s) doesn't have an org anymore after being deleted. This will be generated when they log in next time", foundUser.Username, foundUser.Id)
}
if len(foundUser.ActiveOrg.Id) > 0 {
foundUserOrg, err := GetOrg(ctx, foundUser.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org '%s' in delete user: %s", foundUser.ActiveOrg.Id, err)
} else {
if foundUserOrg.SSOConfig.SSORequired && !ArrayContains(foundUser.ValidatedSessionOrgs, foundUserOrg.Id) {
log.Printf("[AUDIT] User %s (%s) does not have an active session in org with forced SSO %s, so forcing a re-login (aka logout).", foundUser.Username, foundUser.Id, foundUser.ActiveOrg.Id)
foundUser.Session = ""
foundUser.ValidatedSessionOrgs = []string{}
}
}
}
err = SetUser(ctx, foundUser, false)
if err != nil {
log.Printf("[WARNING] Failed removing user %s (%s) from org %s: %s", foundUser.Username, foundUser.Id, userInfo.ActiveOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
// log.Printf("user active organization is %v: ", userInfo)
org, err := GetOrg(ctx, userInfo.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org '%s' in delete user: %s", userInfo.ActiveOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
users := []User{}
for _, user := range org.Users {
if user.Id == foundUser.Id {
continue
}
users = append(users, user)
}
org.Users = users
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org (delete user %s) %s: %s", foundUser.Username, org.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Removed their access but failed updating own user list"}`)))
return
}
log.Printf("[AUDIT] User %s (%s) successfully removed %s from org %s", userInfo.Username, userInfo.Id, foundUser.Username, userInfo.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func HandleDeleteUsersAccountPermanent(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting User request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
userInfo, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in delete user: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "API authentication fail"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var userId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
userId = location[4]
}
ctx := GetContext(request)
userId, err := url.QueryUnescape(userId)
if err != nil {
log.Printf("[WARNING] Failed decoding user %s: %s", userId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
foundUser, err := GetUser(ctx, userId)
if err != nil {
log.Printf("[WARNING] Can't find user %s (delete user): %s", userId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't find user with uername: %v"}`, userInfo.Username)))
return
}
if (userInfo.Id != foundUser.Id) && !userInfo.SupportAccess {
log.Printf("[INFO] Unauthorized user (%s) attempted to delete an account. Must be a user or have support access.", userInfo.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Unauthorize User. Must be a regular user or have support access"}`))
return
}
if !userInfo.SupportAccess {
var requestBody struct {
Password string `json:"password"`
}
if err := json.NewDecoder(request.Body).Decode(&requestBody); err != nil {
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed decoding request body"}`))
return
}
password := requestBody.Password
err = bcrypt.CompareHashAndPassword([]byte(foundUser.Password), []byte(password))
if err != nil {
// Passwords don't match
log.Printf("[WARNING] Password is incorrect for user while deleting account %s (%s): %s", userInfo.Username, userInfo.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Password is incorrect"}`))
return
}
}
// Overwrite incase the user is in the same org
// This could be a way to jump into someone elses organisation if the user already has the correct org name without correct name.
if foundUser.ActiveOrg.Id == "" && foundUser.ActiveOrg.Name == userInfo.ActiveOrg.Name && len(foundUser.Orgs) == 0 {
foundUser.ActiveOrg.Id = string(userInfo.ActiveOrg.Id)
}
if foundUser.SupportAccess {
log.Printf("[AUDIT] Can't delete support user %s (%s)", userInfo.Username, userInfo.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't delete support user"}`))
return
}
orgFound := false
if userInfo.ActiveOrg.Id == foundUser.ActiveOrg.Id {
orgFound = true
} else {
for _, item := range foundUser.Orgs {
if item == userInfo.ActiveOrg.Id {
orgFound = true
break
}
}
}
// FIXME: Add a way to check if the user is a part of the
if !orgFound && !userInfo.SupportAccess {
log.Printf("[AUDIT] User %s (%s) is admin, but can't delete users outside their own org.", userInfo.Username, userInfo.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't change users outside your org (1)."}`)))
return
}
if len(foundUser.Orgs) == 0 {
log.Printf("[INFO] User %s (%s) doesn't have an org anymore after being deleted. This will be generated when they log in next time", foundUser.Username, foundUser.Id)
}
err = SetUser(ctx, foundUser, false)
if err != nil {
log.Printf("[WARNING] Failed removing user %s (%s) from org %s: %s", foundUser.Username, foundUser.Id, userInfo.ActiveOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
// log.Printf("user active organization is %v: ", userInfo)
org, err := GetOrg(ctx, userInfo.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org '%s' in delete user: %s", userInfo.ActiveOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false}`)))
return
}
users := []User{}
for _, user := range org.Users {
if user.Id == foundUser.Id {
continue
}
users = append(users, user)
}
org.Users = users
if len(org.Users) > 1 {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org (delete user %s) %s: %s", foundUser.Username, org.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Removed their access but failed updating own user list"}`)))
return
}
}
err = DeleteUsersAccount(ctx, foundUser)
if err != nil {
log.Printf("[Error] Can't Delete User with User name: %v and Id: %v", foundUser.Username, foundUser.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason":"Can't Delete User with Username: %v}`, userInfo.Username)))
return
}
log.Printf("[AUDIT] User %s (%s) successfully deleted %s (%s)", userInfo.Username, userInfo.Id, foundUser.Username, foundUser.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
// Only used for onprem :/
func UpdateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting App Config Update request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in get all apps: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to edit apps")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
ctx := GetContext(request)
app, err := GetApp(ctx, fileId, user, false)
if err != nil {
log.Printf("[WARNING] Error getting app (update app): %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != app.Owner {
log.Printf("[WARNING] Wrong user (%s) for app %s in update app", user.Username, app.Name)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read in update app: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Public means it's literally public to anyone right away.
type updatefields struct {
Sharing bool `json:"sharing"`
SharingConfig string `json:"sharing_config"`
Public bool `json:"public"`
}
var tmpfields updatefields
err = json.Unmarshal(body, &tmpfields)
if err != nil {
log.Printf("[WARNING] Error with unmarshal body in update app: %s\n%s", err, string(body))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpfields.Sharing != app.Sharing {
log.Printf("[INFO] Changing app sharing for %s to %t", app.ID, tmpfields.Sharing)
app.Sharing = tmpfields.Sharing
if project.Environment != "cloud" {
log.Printf("[INFO] Set app %s (%s) to share everywhere (PUBLIC=true/false), because running onprem", app.Name, app.ID)
app.Public = app.Sharing
}
}
if tmpfields.SharingConfig != app.SharingConfig {
log.Printf("[INFO] Changing app sharing CONFIG for %s to %s", app.ID, tmpfields.SharingConfig)
app.SharingConfig = tmpfields.SharingConfig
}
if tmpfields.Public != app.Public {
log.Printf("[INFO] Changing app %s to PUBLIC (THIS IS DEACTIVATED!)", app.ID)
//app.Public = tmpfields.Public
}
err = SetWorkflowAppDatastore(ctx, *app, app.ID)
if err != nil {
log.Printf("[WARNING] Failed patching workflowapp: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
changed := false
for index, privateApp := range user.PrivateApps {
if privateApp.ID == app.ID {
user.PrivateApps[index] = *app
changed = true
break
}
}
if changed {
err = SetUser(ctx, &user, true)
if err != nil {
log.Printf("[WARNING] Failed updating privateapp %s for user %s: %s", app.ID, user.Username, err)
}
}
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
DeleteCache(ctx, cacheKey)
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
log.Printf("[INFO] Changed App configuration for %s (%s)", app.Name, app.ID)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func deactivateApp(ctx context.Context, user User, app *WorkflowApp) error {
//log.Printf("Should deactivate app %s\n for user %s", app, user)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[DEBUG] Failed getting org %s: %s", user.ActiveOrg.Id, err)
return err
}
if !ArrayContains(org.ActiveApps, app.ID) {
log.Printf("[WARNING] App %s isn't active for org %s", app.ID, user.ActiveOrg.Id)
return errors.New(fmt.Sprintf("App %s isn't active for this org.", app.ID))
}
newApps := []string{}
for _, appId := range org.ActiveApps {
if appId == app.ID {
continue
}
newApps = append(newApps, appId)
}
org.ActiveApps = newApps
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org (deactive app %s) %s: %s", app.ID, org.Id, err)
return err
}
return nil
}
// deleteGCPCloudFunction deletes the GCP Cloud Function associated with the app.
// functionName must be the lowercase function name extracted from app.ReferenceUrl.
// Uses Application Default Credentials — same as deployCloudFunctionPython.
func deleteGCPCloudFunction(ctx context.Context, app *WorkflowApp, functionName string) error {
region := os.Getenv("SHUFFLE_GCE_LOCATION")
if len(region) == 0 {
region = "europe-west2"
}
if len(gceProject) == 0 {
return fmt.Errorf("SHUFFLE_GCEPROJECT not set; cannot delete Cloud Function for app %s (%s)", app.Name, app.ID)
}
service, err := cloudfunctions.NewService(ctx)
if err != nil {
return fmt.Errorf("failed to create Cloud Functions service for app %s (%s): %w", app.Name, app.ID, err)
}
functionPath := fmt.Sprintf("projects/%s/locations/%s/functions/%s", gceProject, region, functionName)
_, err = cloudfunctions.NewProjectsLocationsFunctionsService(service).Delete(functionPath).Do()
if err != nil {
// 404 means the function was never deployed or was already deleted — not an error.
var apiErr *googleapi.Error
if errors.As(err, &apiErr) && apiErr.Code == 404 {
log.Printf("[INFO] Cloud Function '%s' not found (already deleted or never deployed) for app %s (%s)", functionPath, app.Name, app.ID)
return nil
}
return fmt.Errorf("failed to delete Cloud Function '%s' for app %s (%s): %w", functionPath, app.Name, app.ID, err)
}
log.Printf("[INFO] Deleted Cloud Function '%s' for app %s (%s)", functionPath, app.Name, app.ID)
return nil
}
// deleteAppBucketFiles deletes all GCS bucket objects that were written during
// verifySwagger for the given app:
//
// - generated_cloudfunctions/{functionName}.zip
// - generated_apps/{title}_{md5}/*
// - extra_specs/{app.ID}
//
// functionName is the lowercase identifier ("{title}-{md5}") extracted from app.ReferenceUrl.
func deleteAppBucketFiles(ctx context.Context, app *WorkflowApp, functionName string) {
if len(functionName) == 0 {
log.Printf("[WARNING] Empty functionName for bucket cleanup of app %s (%s), skipping", app.Name, app.ID)
return
}
bucketName := fmt.Sprintf("%s.appspot.com", gceProject)
storageClient, err := storage.NewClient(ctx)
if err != nil {
log.Printf("[WARNING] Failed to create storage client for bucket cleanup of app %s (%s): %s", app.Name, app.ID, err)
return
}
defer storageClient.Close()
bucket := storageClient.Bucket(bucketName)
// Delete generated_cloudfunctions/{functionName}.zip directly — no listing needed.
cfPath := fmt.Sprintf("generated_cloudfunctions/%s.zip", functionName)
if delErr := bucket.Object(cfPath).Delete(ctx); delErr != nil {
if !errors.Is(delErr, storage.ErrObjectNotExist) {
log.Printf("[WARNING] Failed to delete bucket object '%s' for app %s (%s): %s", cfPath, app.Name, app.ID, delErr)
}
} else {
log.Printf("[INFO] Deleted bucket object '%s' for app %s (%s)", cfPath, app.Name, app.ID)
}
// Delete all files under generated_apps/{title}_{md5}/ using an exact prefix so we
// only list this app's directory rather than the entire generated_apps/ tree.
// functionName = "{title}-{md5}" — strip the trailing "-{32-char md5}" to get title.
if len(functionName) >= 33 {
title := functionName[:len(functionName)-33]
md5Hash := functionName[len(functionName)-32:]
appsPrefix := fmt.Sprintf("generated_apps/%s_%s/", title, md5Hash)
it := bucket.Objects(ctx, &storage.Query{Prefix: appsPrefix})
for {
attrs, iterErr := it.Next()
if iterErr == iterator.Done {
break
}
if iterErr != nil {
log.Printf("[WARNING] Failed iterating bucket objects with prefix '%s' for app %s (%s): %s", appsPrefix, app.Name, app.ID, iterErr)
break
}
if delErr := bucket.Object(attrs.Name).Delete(ctx); delErr != nil {
log.Printf("[WARNING] Failed to delete bucket object '%s' for app %s (%s): %s", attrs.Name, app.Name, app.ID, delErr)
} else {
log.Printf("[INFO] Deleted bucket object '%s' for app %s (%s)", attrs.Name, app.Name, app.ID)
}
}
}
// Delete extra_specs/{app.ID} — used when app data is too large for datastore.
extraSpecsPath := fmt.Sprintf("extra_specs/%s", app.ID)
if delErr := bucket.Object(extraSpecsPath).Delete(ctx); delErr != nil {
if !errors.Is(delErr, storage.ErrObjectNotExist) {
log.Printf("[WARNING] Failed to delete bucket object '%s' for app %s (%s): %s", extraSpecsPath, app.Name, app.ID, delErr)
}
} else {
log.Printf("[INFO] Deleted bucket object '%s' for app %s (%s)", extraSpecsPath, app.Name, app.ID)
}
}
// deleteAppCloudResources is the entry point called from DeleteWorkflowApp.
// It parses the app's ReferenceUrl, then delegates to deleteGCPCloudFunction
// and deleteAppBucketFiles.
func deleteAppCloudResources(ctx context.Context, app *WorkflowApp) error {
if !strings.Contains(app.ReferenceUrl, "cloudfunctions.net") {
log.Printf("[DEBUG] App %s (%s) has no Cloud Function URL, skipping GCP cleanup", app.Name, app.ID)
return nil
}
parsedURL, err := url.Parse(app.ReferenceUrl)
if err != nil {
return fmt.Errorf("failed to parse Cloud Function URL '%s' for app %s (%s): %w", app.ReferenceUrl, app.Name, app.ID, err)
}
pathParts := strings.Split(strings.TrimPrefix(parsedURL.Path, "/"), "/")
if len(pathParts) == 0 || pathParts[0] == "" {
return fmt.Errorf("could not extract function name from URL path '%s' for app %s (%s)", parsedURL.Path, app.Name, app.ID)
}
// functionName is strings.ToLower(identifier) where identifier = "{title}-{md5}".
// The MD5 (32 hex chars) is always the last 32 chars of functionName.
functionName := strings.ToLower(pathParts[0])
// Always attempt bucket cleanup even if the Cloud Function is already gone or fails to delete.
if err := deleteGCPCloudFunction(ctx, app, functionName); err != nil {
log.Printf("[WARNING] Cloud Function deletion failed for app %s (%s), continuing with bucket cleanup: %s", app.Name, app.ID, err)
}
deleteAppBucketFiles(ctx, app, functionName)
return nil
}
func DeleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in delete app: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to delete apps: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
ctx := GetContext(request)
app, err := GetApp(ctx, fileId, user, false)
if err != nil {
log.Printf("[WARNING] Error getting app %s: %s", app.Name, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if project.Environment != "cloud" && len(app.ReferenceOrg) == 0 {
app.ReferenceOrg = user.ActiveOrg.Id
}
if user.Id != app.Owner && app.ReferenceOrg != user.ActiveOrg.Id {
if user.Role == "admin" && app.Owner == "" {
log.Printf("[INFO] Anyone can edit %s (%s), since it doesn't have an owner (DELETE).", app.Name, app.ID)
} else {
if user.Role == "admin" {
err = deactivateApp(ctx, user, app)
if err == nil {
log.Printf("[INFO] App %s was deactivated for org %s", app.ID, user.ActiveOrg.Id)
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-100"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-500"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-1000"))
DeleteCache(ctx, "all_apps")
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
return
}
}
log.Printf("[WARNING] Wrong user (%s) for app %s (%s) when DELETING app", user.Username, app.Name, app.ID)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "You need to be admin to deactivate apps for an org."}`))
return
}
}
if (app.Public || app.Sharing) && project.Environment == "cloud" {
log.Printf("[WARNING] App %s being deleted is public. Shouldn't be allowed. Public: %t, Sharing: %t", app.Name, app.Public, app.Sharing)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Can't delete public apps. Unpublish. Contact support@shuffler.io if you encounter any problem."}`))
return
}
if len(app.ReferenceUrl) > 0 && project.Environment == "cloud" {
appCopy := *app
go func() {
const maxAttempts = 3
baseDelay := 2 * time.Second
for attempt := 1; attempt <= maxAttempts; attempt++ {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
err := deleteAppCloudResources(ctx, &appCopy)
cancel()
if err == nil {
return
}
if attempt == maxAttempts {
log.Printf("[WARNING] Background GCP cleanup failed for app %s (%s) after %d attempts: %s", appCopy.Name, appCopy.ID, maxAttempts, err)
return
}
log.Printf("[WARNING] Background GCP cleanup attempt %d/%d failed for app %s (%s): %s — retrying in %s", attempt, maxAttempts, appCopy.Name, appCopy.ID, err, baseDelay)
time.Sleep(baseDelay)
baseDelay *= 2
}
}()
}
// Not really deleting it, just removing from user cache
var privateApps []WorkflowApp
for _, item := range user.PrivateApps {
if item.ID == fileId {
continue
}
privateApps = append(privateApps, item)
}
user.PrivateApps = privateApps
err = SetUser(ctx, &user, true)
if err != nil {
log.Printf("[WARNING] Failed removing %s app for user %s: %s", app.Name, user.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false"}`)))
return
}
err = DeleteKey(ctx, "workflowapp", app.ID)
if err != nil {
log.Printf("[WARNING] Failed deleting %s (%s) for by %s: %s", app.Name, app.ID, user.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false"}`)))
return
}
// This is getting stupid :)
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-100"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-500"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-1000"))
DeleteCache(ctx, "all_apps")
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func HandleKeyValueCheck(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
// Append: Checks if the value should be appended
// WorkflowCheck: Checks if the value should only check for workflow in org, or entire org
// Authorization: the Authorization to use
// ExecutionRef: Ref for the execution
// Values: The values to use
type DataValues struct {
App string
Actions string
ParameterNames []string
ParameterValues []string
}
type ReturnData struct {
Append bool `json:"append"`
WorkflowCheck bool `json:"workflow_check"`
Authorization string `json:"authorization"`
ExecutionRef string `json:"execution_ref"`
OrgId string `json:"org_id"`
Values []DataValues `json:"values"`
}
//for key, value := range data.Apps {
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling test: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpData.OrgId != fileId {
log.Printf("[INFO] OrgId %s and %s don't match (key value check)", tmpData.OrgId, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Organization ID's don't match"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[INFO] Organization %s doesn't exist: %s", tmpData.OrgId, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
workflowExecution, err := GetWorkflowExecution(ctx, tmpData.ExecutionRef)
if err != nil {
log.Printf("[INFO] Couldn't find workflow execution: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "No permission to get execution"}`))
return
}
if workflowExecution.Authorization != tmpData.Authorization {
log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "Auth doesn't match"}`))
return
}
if workflowExecution.Status != "EXECUTING" {
log.Printf("[INFO][%s] Workflow isn't executing and shouldn't be getting an app key", workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Workflow isn't executing (1)"}`))
return
}
if workflowExecution.ExecutionOrg != org.Id {
log.Printf("[INFO] Org %s wasn't used to execute %s", org.Id, workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Bad organization specified"}`))
return
}
if len(tmpData.Values) != 1 {
log.Printf("[INFO] Filter data can only hande 1 value right now, not %d", len(tmpData.Values))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "Can't handle multiple apps yet, just one"}`))
return
}
value := tmpData.Values[0]
// FIXME: Alphabetically sort the parameternames
// FIXME: Add organization wide search, not just workflow based
found := []string{}
notFound := []string{}
dbKey := fmt.Sprintf("app_execution_values")
parameterNames := fmt.Sprintf("%s_%s", value.App, strings.Join(value.ParameterNames, "_"))
if tmpData.WorkflowCheck {
// FIXME: Make this alphabetical
for _, value := range value.ParameterValues {
if len(value) == 0 {
log.Printf("Shouldn't have value of length 0!")
continue
}
log.Printf("[INFO] Looking for value %s in Workflow %s of ORG %s", value, workflowExecution.Workflow.ID, org.Id)
executionValues, err := GetAppExecutionValues(ctx, parameterNames, org.Id, workflowExecution.Workflow.ID, value)
if err != nil {
log.Printf("[WARNING] Failed getting key %s: %s (1)", dbKey, err)
notFound = append(notFound, value)
//found = append(found, value)
continue
}
foundCount := len(executionValues)
if foundCount > 0 {
found = append(found, value)
} else {
log.Printf("[INFO] Found for %s: %d", dbKey, foundCount)
notFound = append(notFound, value)
}
}
} else {
log.Printf("[INFO] Should validate if value %s is in workflow id %s", value, workflowExecution.Workflow.ID)
for _, value := range value.ParameterValues {
if len(value) == 0 {
log.Printf("Shouldn't have value of length 0!")
continue
}
log.Printf("[INFO] Looking for value %s in ORG %s", value, org.Id)
executionValues, err := GetAppExecutionValues(ctx, parameterNames, org.Id, workflowExecution.Workflow.ID, value)
if err != nil {
log.Printf("[WARNING] Failed getting key %s: %s (2)", dbKey, err)
notFound = append(notFound, value)
//found = append(found, value)
continue
}
foundCount := len(executionValues)
if foundCount > 0 {
found = append(found, value)
} else {
log.Printf("[INFO] Found for %s: %d", dbKey, foundCount)
notFound = append(notFound, value)
}
}
}
//App string
//Actions string
//ParameterNames string
//ParamererValues []string
appended := 0
if tmpData.Append {
log.Printf("[INFO] Should append %d value(s) in K:V for %s_%s!", len(notFound), org.Id, workflowExecution.ExecutionId)
//parameterNames := strings.Join(value.ParameterNames, "_")
for _, notFoundValue := range notFound {
newRequest := NewValue{
OrgId: org.Id,
WorkflowExecutionId: workflowExecution.ExecutionId,
ParameterName: parameterNames,
Value: notFoundValue,
}
// WorkflowId: workflowExecution.Workflow.Id,
if tmpData.WorkflowCheck {
newRequest.WorkflowId = workflowExecution.Workflow.ID
}
err = SetNewValue(ctx, newRequest)
if err != nil {
log.Printf("[ERROR] Error adding %s to appvalue: %s", notFoundValue, err)
continue
}
appended += 1
log.Printf("[INFO] Added %s as new appvalue to datastore", notFoundValue)
}
}
type returnStruct struct {
Success bool `json:"success"`
Appended int `json:"appended"`
Found []string `json:"found"`
}
returnData := returnStruct{
Success: true,
Appended: appended,
Found: found,
}
b, _ := json.Marshal(returnData)
resp.WriteHeader(200)
resp.Write(b)
}
// Used for swapping your own organization to a new one IF it's eligible
func HandleChangeUserOrg(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just getting here for later
ctx := GetContext(request)
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in change org (local): %s", userErr)
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// Clean up the users' cache for different parts
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey))
DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey))
DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session))
log.Printf("[DEBUG] Redirecting ORGCHANGE request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey))
DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey))
DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session))
return
}
}
if userErr != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type ReturnData struct {
OrgId string `json:"org_id" datastore:"org_id"`
RegionUrl string `json:"region_url" datastore:"region_url"`
// SSO bool `json:"sso"`
SSOTest bool `json:"sso_test"`
SSO bool `json:"sso"`
Mode string `json:"mode"`
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
if len(fileId) == 36 && strings.Count(fileId, "-") == 4 {
log.Printf("[DEBUG] Empty body in change org, using fileId from URL: %s", fileId)
tmpData.OrgId = fileId
} else {
log.Printf("[WARNING] Failed unmarshalling change org body: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if tmpData.SSOTest || tmpData.SSO {
tmpData.SSOTest = true
tmpData.SSO = true
}
foundOrg := false
for _, org := range user.Orgs {
if org == tmpData.OrgId {
foundOrg = true
break
}
}
if user.ActiveOrg.Id == fileId && tmpData.SSO == false {
log.Printf("[WARNING] User swap to the org \"%s\" - already in the org", tmpData.OrgId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "You are already in that organisation"}`))
return
}
// Support access pivot to an org BY EMAIL (not org id)
if strings.Contains(fileId, "@") && strings.Contains(fileId, ".") && user.SupportAccess && user.Active && user.Verified && project.Environment == "cloud" {
//foundUsers, err := FindUser(ctx, fileId)
foundUsers, err := FindUser(ctx, fileId)
if err != nil || len(foundUsers) == 0 {
log.Printf("[ERROR] Failed finding user %s for support access: %s", user.Username, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed finding user for support access"}`))
return
}
if len(foundUsers) > 1 {
log.Printf("[ERROR] Found multiple users for support access user %s", user.Username)
}
newUsers := []User{}
for _, loopUser := range foundUsers {
if strings.ToLower(strings.TrimSpace(loopUser.Username)) != fileId {
continue
}
newUsers = append(newUsers, loopUser)
}
if len(newUsers) == 0 {
log.Printf("[WARNING] No user found with username '%s' for support access user %s", fileId, user.Username)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No user found with that username"}`))
return
}
foundUsers = newUsers
log.Printf("[AUDIT] Support access user %s is trying to swap to org for user %s (%s). Found ID: %s", user.Username, fileId, foundUsers[0].Id, foundUsers[0].ActiveOrg.Id)
tmpData.OrgId = foundUsers[0].ActiveOrg.Id
fileId = foundUsers[0].ActiveOrg.Id
}
// Add instantswap of backend
// This could in theory be built out open source as well
regionUrl := ""
if project.Environment == "cloud" && user.SupportAccess {
regionUrl = "https://shuffler.io"
foundOrg = true
}
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[WARNING] Organization %s doesn't exist: %s", tmpData.OrgId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if !foundOrg || tmpData.OrgId != fileId {
found := false
if !foundOrg {
for _, user := range org.Users {
if user.Id == user.Id {
log.Printf("[ERROR] User %s (%s) lost org %s (%s) in their user list, but has it in their org list. Fixing.", user.Username, user.Id, org.Name, org.Id)
user.Orgs = append(user.Orgs, org.Id)
found = true
break
}
}
}
if !found {
log.Printf("[WARNING] User swap to the org \"%s\" - access denied", tmpData.OrgId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "No permission to change to this org. Please contact support@shuffler.io if this is unexpected."}`))
return
}
}
if (org.SSOConfig.SSORequired == true && user.UsersLastSession != user.Session && user.SupportAccess == false) || tmpData.SSO {
// Check if the org is the suborg or not?
skipSSO := false
if len(org.CreatorOrg) > 0 {
log.Printf("[DEBUG] User %s (%s) is trying to change to suborg %s (%s)", user.Username, user.Id, org.Name, org.Id)
parentOrg, err := GetOrg(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[ERROR] Failed getting parent org %s for suborg %s: %s", org.CreatorOrg, org.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting parent org for suborg"}`))
return
}
if parentOrg.SSOConfig.SkipSSOForAdmins {
for _, orgUser := range parentOrg.Users {
if orgUser.Id == user.Id && orgUser.Role == "admin" {
log.Printf("[DEBUG] User %s (%s) is admin in parent org %s (%s) and can skip SSO", user.Username, user.Id, parentOrg.Name, parentOrg.Id)
// Skip SSO for admin in suborgs
skipSSO = true
break
}
}
}
}
baseSSOUrl := ""
redirectKey := "SSO_REDIRECT"
if len(org.SSOConfig.OpenIdAuthorization) > 0 {
log.Printf("[INFO] OpenID login for %s", org.Id)
redirectKey = "SSO_REDIRECT"
baseSSOUrl = GetOpenIdUrl(request, *org)
}
if skipSSO || len(baseSSOUrl) == 0 {
log.Printf("[AUDIT] User %s (%s) is skipping SSO for suborg %s (%s). URL: %s", user.Username, user.Id, org.Name, org.Id, baseSSOUrl)
} else {
if !strings.HasPrefix(baseSSOUrl, "http") {
log.Printf("[ERROR] SSO URL for %s (%s) is invalid: %s", org.Name, org.Id, baseSSOUrl)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "SSO URL is invalid"}`))
return
} else {
// Check if the user has other orgs that can be swapped to - if so SWAP
log.Printf("[DEBUG] Change org: Should redirect user %s in org %s (%s) to SSO login at %s", user.Username, user.ActiveOrg.Name, user.ActiveOrg.Id, baseSSOUrl)
ssoResponse := SSOResponse{
Success: true,
Reason: redirectKey,
URL: baseSSOUrl,
}
b, err := json.Marshal(ssoResponse)
if err != nil {
log.Printf("[ERROR] Failed marshalling SSO response: %s", err)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(b)
return
}
}
}
if project.Environment == "cloud" && len(org.RegionUrl) > 0 && !strings.Contains(org.RegionUrl, "\"") {
regionUrl = org.RegionUrl
}
if len(regionUrl) > 0 && !ArrayContains(user.Regions, regionUrl) {
user.Regions = append(user.Regions, regionUrl)
}
userFound := false
usr := User{}
for _, orgUsr := range org.Users {
if user.Id == orgUsr.Id {
usr = orgUsr
userFound = true
break
}
}
if !userFound && !user.SupportAccess {
// FIXME: This changes the source of truth from JUST org.Users to user.Orgs
// May be a problem in worst case scenarios, but only works for orgids
// you know, so chance of causing an issue is **VERY** low.
found := false
for _, orgId := range user.Orgs {
if orgId == org.Id {
usr.Role = "user"
found = true
break
}
}
if !found {
log.Printf("[ERROR] User %s (%s) can't change to org %s (%s) (2)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "No permission to change to this org (2). Please contact support@shuffler.io if this is unexpected."}`))
return
}
}
if user.SupportAccess {
usr.Role = "admin"
user.Role = "admin"
}
user.ActiveOrg = OrgMini{
Name: org.Name,
Id: org.Id,
Role: usr.Role,
}
user.Role = usr.Role
err = SetUser(ctx, &user, false)
if err != nil {
log.Printf("[ERROR] Failed updating user when changing org: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
expiration := time.Now().Add(8 * time.Hour)
newCookie := ConstructSessionCookie(user.Session, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
// Cleanup cache for the user
DeleteCache(ctx, fmt.Sprintf("%s_workflows", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey))
DeleteCache(ctx, fmt.Sprintf("%s", user.ApiKey))
DeleteCache(ctx, user.Session)
DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session))
log.Printf("[INFO] User %s (%s) successfully changed org to '%s' (%s)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Changed Organization", "region_url": "%s", "org_id": "%s"}`, regionUrl, org.Id)))
}
func HandleCreateSubOrg(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in creating sub org: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Can't make suborg without being admin: %s (%s).", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
ctx := GetContext(request)
parentOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Organization %s doesn't exist or failed to load: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Just for cache reseting across regions
for _, inneruser := range parentOrg.Users {
DeleteCache(ctx, inneruser.ApiKey)
DeleteCache(ctx, inneruser.Session)
DeleteCache(ctx, fmt.Sprintf("session_%s", inneruser.Session))
DeleteCache(ctx, fmt.Sprintf("user_%s", inneruser.Id))
DeleteCache(ctx, fmt.Sprintf("%s_workflows", inneruser.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", inneruser.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", inneruser.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("user_%s", inneruser.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", inneruser.Id))
// Added another to ensure we handle empty cursor
DeleteCache(ctx, fmt.Sprintf("%s__childorgs", inneruser.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("%s_childorgs", inneruser.ActiveOrg.Id))
}
// Delete parent org cache as well from the org region
DeleteCache(ctx, fmt.Sprintf("Organizations_%s", parentOrg.Id))
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Create Suborg request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body in create suborg: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type ReturnData struct {
OrgId string `json:"org_id" datastore:"org_id"`
OrgName string `json:"org_name" datastore:"org_name"`
Name string `json:"name" datastore:"name"`
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[INFO] Failed unmarshalling test: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "The data is badly formatted"}`))
return
}
if len(tmpData.OrgName) > 0 && len(tmpData.Name) == 0 {
tmpData.Name = tmpData.OrgName
}
if len(tmpData.Name) < 3 {
log.Printf("[WARNING] Suborgname too short (min 3) %s", tmpData.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Name must at least be 3 characters. Required fields: org_id, name"}`))
return
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if tmpData.OrgId != user.ActiveOrg.Id || fileId != user.ActiveOrg.Id {
log.Printf("[WARNING] User can't edit the org \"%s\"", tmpData.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No permission to edit this org (1). Org Id has to match in the body and the request."}`))
return
}
if len(parentOrg.ManagerOrgs) > 0 {
log.Printf("[WARNING] Organization %s can't have suborgs, as it's as suborg", tmpData.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Can't make suborg of suborg. Switch to a parent org to make another."}`))
return
}
if project.Environment == "cloud" {
//if !parentOrg.SyncFeatures.MultiTenant.Active && !parentOrg.LeadInfo.Customer && !parentOrg.LeadInfo.POV && !parentOrg.LeadInfo.Internal {
parentOrg.SyncFeatures.MultiTenant.Active = true
// Anyone is allowed to make 5
baseLimit := int64(2)
baseCustomerLimit := int64(5)
if parentOrg.SyncFeatures.MultiTenant.Limit <= baseLimit {
parentOrg.SyncFeatures.MultiTenant.Limit = baseLimit
}
if parentOrg.LeadInfo.Customer || parentOrg.LeadInfo.Internal || parentOrg.LeadInfo.POV {
if parentOrg.SyncFeatures.MultiTenant.Limit < baseCustomerLimit {
parentOrg.SyncFeatures.MultiTenant.Limit = baseCustomerLimit
}
}
if parentOrg.SyncUsage.MultiTenant.Counter >= parentOrg.SyncFeatures.MultiTenant.Limit {
log.Printf("[WARNING] Org %s is not allowed to make more than %d sub-organizations.", parentOrg.Id, parentOrg.SyncFeatures.MultiTenant.Limit)
resp.WriteHeader(400)
//resp.Write([]byte(`{"success": false, "reason": "Sub-organizations require an active subscription or to be in the POV stage with access to multi-tenancy. Contact support@shuffler.io to try it out."}`))
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You have made %d/%d sub-organizations. Contact support@shuffler.io to increase this limit"}`, parentOrg.SyncUsage.MultiTenant.Counter, parentOrg.SyncFeatures.MultiTenant.Limit)))
return
}
//}
parentOrg.SyncUsage.MultiTenant.Counter += 1
log.Printf("[DEBUG] Allowing suborg for %s because they have %d vs %d limit", parentOrg.Id, len(parentOrg.ChildOrgs), parentOrg.SyncFeatures.MultiTenant.Limit)
} else {
//log.Printf("MULTITENANT USAGE: %d / %d. Active: %#v", parentOrg.SyncUsage.MultiTenant.Counter, parentOrg.SyncFeatures.MultiTenant.Limit, parentOrg.SyncFeatures.MultiTenant.Active)
childOrgs, _, err := GetAllChildOrgs(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting child orgs for %s: %s", user.ActiveOrg.Id, err)
}
if len(childOrgs) > 0 {
parentOrg.SyncUsage.MultiTenant.Counter = int64(len(childOrgs))
}
license := checkNoInternet()
isLicensed := license.Valid
if !parentOrg.CloudSync && !isLicensed && len(childOrgs) >= 3 {
log.Printf("[WARNING] Organization %s has exceeded the free plan limit of 3 sub-organizations. An enterprise license is required to create additional sub-organizations.", parentOrg.Id)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "The free plan allows up to 3 sub-organizations. To create more, please upgrade to an enterprise license or contact support@shuffler.io for more information"}`))
return
}
org := HandleCheckLicense(ctx, *parentOrg)
parentOrg = &org
if parentOrg.SyncFeatures.MultiTenant.Active && parentOrg.SyncUsage.MultiTenant.Counter >= parentOrg.SyncFeatures.MultiTenant.Limit {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You have made %d/%d sub-organizations. To create more, please upgrade to an enterprise license or contact support@shuffler.io for more information"}`, parentOrg.SyncUsage.MultiTenant.Counter, parentOrg.SyncFeatures.MultiTenant.Limit)))
return
}
}
orgId := uuid.NewV4().String()
newApps := parentOrg.ActiveApps
if len(newApps) > 11 {
// Do the last 10 apps, not 10 first
newApps = newApps[len(newApps)-10:]
}
newOrg := Org{
Name: tmpData.Name,
Description: fmt.Sprintf("Sub-org by user %s in parent-org %s", user.Username, parentOrg.Name),
Image: parentOrg.Image,
Id: orgId,
Org: tmpData.Name,
Users: []User{
user,
},
Roles: []string{"admin", "user"},
CloudSync: false,
ManagerOrgs: []OrgMini{
OrgMini{
Id: tmpData.OrgId,
Name: parentOrg.Name,
},
},
CloudSyncActive: parentOrg.CloudSyncActive,
CreatorOrg: tmpData.OrgId,
Region: parentOrg.Region,
RegionUrl: parentOrg.RegionUrl,
Defaults: parentOrg.Defaults,
// FIXME: Should this be here? Makes things slow~
// Should only append apps owned by the parentorg itself
ActiveApps: newApps,
}
// FIXME: This may be good to auto distribute no matter what
// Then maybe the kms problem won't happen
parentOrg.ChildOrgs = append(parentOrg.ChildOrgs, OrgMini{
Name: tmpData.Name,
Id: orgId,
})
DeleteCache(ctx, fmt.Sprintf("Organizations_%s", parentOrg.Id))
err = SetOrg(ctx, *parentOrg, parentOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed updating parent org %s: %s", newOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Update all admins to have access to this suborg
for _, loopUser := range parentOrg.Users {
if loopUser.Role != "admin" {
continue
}
//if loopUser.Id == user.Id {
// continue
//}
foundUser, err := GetUser(ctx, loopUser.Id)
if err != nil {
log.Printf("[ERROR] User with Identifier %s doesn't exist: %s (update admins - create)", loopUser.Id, err)
continue
}
// Random between 50-200ms
mathrand.Seed(time.Now().UnixNano())
randInt := mathrand.Intn(150)
time.Sleep(time.Duration(randInt+50) * time.Millisecond)
// Add org to user
foundUser.Orgs = append(foundUser.Orgs, newOrg.Id)
err = SetUser(ctx, foundUser, false)
if err != nil {
log.Printf("[ERROR] Failed updating user when setting creating suborg (update admins - update): %s ", err)
continue
}
// Add user to org
newOrg.Users = append(newOrg.Users, loopUser)
}
err = SetOrg(ctx, newOrg, newOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed setting new org %s: %s", newOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Re-read from DB to avoid writing back stale cached user.Orgs
// (HandleApiAuthentication may have served a cached copy missing recent org additions)
freshUser, freshErr := GetUser(ctx, user.Id)
if freshErr == nil {
user = *freshUser
}
if !ArrayContains(user.Orgs, newOrg.Id) {
user.Orgs = append(user.Orgs, newOrg.Id)
}
err = SetUser(ctx, &user, false)
if err != nil {
log.Printf("[WARNING] Failed updating user when setting creating suborg: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
// 1. Get environments for parent
// 2. Create an environment for the child with the same name
if project.Environment != "cloud" {
environments, err := GetEnvironments(ctx, parentOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting environments for parent org: %s", err)
} else {
defaultFoundEnv := Environment{}
for _, parentEnv := range environments {
if parentEnv.Default {
defaultFoundEnv = parentEnv
break
}
}
if len(defaultFoundEnv.Name) > 0 && defaultFoundEnv.Type != "cloud" {
item := Environment{
Name: defaultFoundEnv.Name,
Type: defaultFoundEnv.Type,
OrgId: newOrg.Id,
Default: true,
Id: uuid.NewV4().String(),
Auth: defaultFoundEnv.Auth,
}
err := SetEnvironment(ctx, &item)
if err != nil {
log.Printf("[ERROR] Failed setting up new environment for new org: %s", err)
} else {
log.Printf("[INFO] Successfully created new parent-duplicated environment for new suborg %s", newOrg.Id)
}
}
}
}
log.Printf("[INFO] User %s SUCCESSFULLY ADDED child org %s (%s) for parent %s (%s)", user.Username, newOrg.Name, newOrg.Id, parentOrg.Name, parentOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s", "reason": "Successfully created new sub-org"}`, newOrg.Id)))
}
func getSignatureSample(org Org) PaymentSubscription {
if len(org.Subscriptions) > 0 {
for _, sub := range org.Subscriptions {
if !sub.EulaSigned {
return sub
}
}
}
//log.Printf("[DEBUG] No signature sample found for org %s", org.Id)
parsedEula := GetOnpremPaidEula()
if (org.LeadInfo.Customer || org.LeadInfo.POV || len(org.ManagerOrgs) > 0) && !org.LeadInfo.OpenSource {
return PaymentSubscription{}
name := "App execution units - default"
if len(org.ManagerOrgs) > 0 {
name = "Suborg access - default"
}
return PaymentSubscription{
Active: true,
Startdate: int64(time.Now().Unix()),
CancellationDate: 0,
Enddate: 0,
Name: name,
Recurrence: string("monthly"),
Amount: string(rune(100000)),
Currency: string("USD"),
Level: "1",
Reference: "TBD",
Limit: 100000,
Features: []string{
"Custom Contract features",
"Multi-Tenant & Multi-Region",
},
EulaSigned: true,
Eula: parsedEula,
}
} else if (org.LeadInfo.Customer || org.LeadInfo.POV) && org.LeadInfo.OpenSource {
name := "Open Source Scale Units"
licensedWorkerUrl := os.Getenv("LICENSED_WORKER_URL")
nightlyWorkerUrl := os.Getenv("NIGHTLY_WORKER_URL")
if !org.EulaSigned {
licensedWorkerUrl = "Sign EULA first"
nightlyWorkerUrl = "Sign EULA first"
}
features := []string{
"Priority Support",
fmt.Sprintf("App and Workflow development support"),
"Documentation: https://shuffler.io/docs/configuration#scaling_shuffle_with_swarm",
fmt.Sprintf("Stable Worker License: %s", licensedWorkerUrl),
fmt.Sprintf("Nightly Worker License: %s", nightlyWorkerUrl),
}
return PaymentSubscription{
Name: name,
Active: true,
CancellationDate: 0,
Enddate: 0,
Startdate: int64(time.Now().Unix()),
Recurrence: string("monthly"),
Amount: string(rune(600)),
Currency: string("USD"),
Level: "1",
Reference: "TBD",
Limit: 1,
Features: features,
EulaSigned: org.EulaSigned,
Eula: parsedEula,
}
}
return PaymentSubscription{}
}
func BuildBaseSubscription(org Org, monthlyExecLimit int64) PaymentSubscription {
now := int64(time.Now().Unix())
log.Printf("[DEBUG] Building base subscription for org %s that has %d monthly exec limit", org.Id, monthlyExecLimit)
// Default values
planName := ""
supportLevel := ""
features := []string{}
amount := "0"
parsedEula := GetOnpremPaidEula()
eulaSigned := false
if project.Environment == "cloud" {
// Cloud licenses
if monthlyExecLimit >= 300000 {
planName = "Cloud Enterprise License"
supportLevel = "Enterprise Support"
features = []string{
"∞ Days Workflow Backup",
"∞ Users",
"Critical Response",
"On-Call Support",
"Setup and Maintenance",
"Key Management System",
"Custom Integrations",
"Custom Scaling Options",
"Billing and Invoice Included",
"Custom Contract",
}
amount = "870" // Just for placeholder
} else if monthlyExecLimit >= 12000 {
planName = "Cloud Scale License"
supportLevel = "Standard Support"
features = []string{
"30 Days workflow run history",
"14 Days workflow backup",
"15 Users",
"Select Datacenter Region",
}
amount = fmt.Sprintf("%d", int64(((monthlyExecLimit-2000)/10000)*32)) // Calculate based on app runs: (paid_runs / 10k) * $32
} else if monthlyExecLimit >= 2000 && monthlyExecLimit < 12000 {
planName = "Free License"
supportLevel = "Community Support"
features = []string{
"All 2500+ Apps",
"All usecase templates",
"1 Day workflow run history",
"7 Days workflow backup",
"5 Users",
}
amount = "0" // Just for placeholder
}
} else {
// Open source licenses
planName = "Open Source License"
supportLevel = "Community Support"
features = []string{
"All 2500+ Apps",
"All usecase templates",
"Custom Workflows",
"1 Hour Workflow Run History",
"1 Hour Workflow Backup",
}
amount = "0" // Just for placeholder
}
t := time.Now().UTC()
firstNextMonth := time.Date(t.Year(), t.Month()+1, 1, 0, 0, 0, 0, time.UTC)
endDate := int64(firstNextMonth.Unix())
return PaymentSubscription{
Id: uuid.NewV4().String(),
Active: true,
Startdate: now,
Enddate: endDate,
CancellationDate: 0,
Name: planName,
SupportLevel: supportLevel,
Recurrence: string("monthly"),
Amount: amount,
Currency: string("USD"),
Level: "1",
Reference: "", // IMPORTANT: empty for base/unpaid
Limit: 0,
Features: features,
EulaSigned: eulaSigned,
Eula: parsedEula,
}
}
func HandleEditOrg(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Edit Org request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in edit org: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Not admin: %s (%s).", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type ReturnData struct {
Tutorial string `json:"tutorial" datastore:"tutorial"`
Name string `json:"name" datastore:"name"`
Image string `json:"image" datastore:"image"`
CompanyType string `json:"company_type" datastore:"company_type"`
Description string `json:"description" datastore:"description"`
OrgId string `json:"org_id" datastore:"org_id"`
Priority string `json:"priority" datastore:"priority"`
Defaults Defaults `json:"defaults" datastore:"defaults"`
SSOConfig SSOConfig `json:"sso_config" datastore:"sso_config"`
LeadInfo []string `json:"lead_info" datastore:"lead_info"`
MFARequired bool `json:"mfa_required" datastore:"mfa_required"`
CreatorConfig string `json:"creator_config" datastore:"creator_config"`
Subscription PaymentSubscription `json:"subscription" datastore:"subscription"`
SubscriptionIndex string `json:"subscription_index" datastore:"subscription_index"`
SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"`
Billing Billing `json:"billing" datastore:"billing"`
Branding OrgBranding `json:"branding" datastore:"branding"`
EditingBranding bool `json:"editing_branding" datastore:"editing_branding"`
Editing string `json:"editing" datastore:"editing"`
}
var tmpData ReturnData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling test: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
//log.Printf("SSO: %s", tmpData.SSOConfig)
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
admin := false
if (tmpData.OrgId != user.ActiveOrg.Id || fileId != user.ActiveOrg.Id) && !tmpData.SyncFeatures.Editing {
log.Printf("[WARNING] User can't edit org %s (active: %s)", fileId, user.ActiveOrg.Id)
if !user.SupportAccess {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No permission to edit this org (2)"}`))
return
}
log.Printf("[AUDIT] User %s (%s) is editing org %s (%s) with support access", user.Username, user.Id, fileId, user.ActiveOrg.Id)
admin = true
}
ctx := GetContext(request)
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[WARNING] Organization doesn't exist: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
userFound := false
for _, inneruser := range org.Users {
if inneruser.Id == user.Id {
userFound = true
if inneruser.Role == "admin" {
admin = true
}
break
}
}
if user.SupportAccess {
log.Printf("[AUDIT] User %s (%s) is editing org %s (%s) with support access", user.Username, user.Id, fileId, user.ActiveOrg.Id)
userFound = true
admin = true
}
if !userFound && !user.SupportAccess {
log.Printf("[WARNING] User %s doesn't exist in organization for edit %s", user.Id, org.Id)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if !admin {
log.Printf("[WARNING] User %s doesn't have edit rights to %s", user.Id, org.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpData.Editing == "subscription_update" && !user.SupportAccess {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Support access required"}`))
return
}
// Allow editing a specific subscription card from UI except Eula and Reference
if tmpData.Editing == "subscription_update" {
// Find subscription by ID (SubscriptionIndex now holds the ID string)
var idx int = -1
for i, sub := range org.Subscriptions {
if sub.Id == tmpData.SubscriptionIndex {
idx = i
break
}
}
if idx == -1 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "subscription not found by ID"}`))
return
}
// Preserve immutable fields
existing := org.Subscriptions[idx]
updated := tmpData.Subscription
updated.Id = existing.Id
updated.Eula = existing.Eula
// Do not overwrite existing EULA signature info if it's already signed
if existing.EulaSigned {
updated.EulaSigned = existing.EulaSigned
updated.EulaSignedBy = existing.EulaSignedBy
} else if len(tmpData.Subscription.Eula) > 0 && tmpData.Subscription.Eula == existing.Eula && tmpData.Subscription.Active {
// This will handle the eula signing
updated.EulaSigned = true
updated.EulaSignedBy = user.Username
}
// Apply the rest
org.Subscriptions[idx] = updated
// This is to set user in free plan is the current plan is Inactive by us
if len(org.CreatorOrg) == 0 {
hasActivePaidSubscription := false
hasFreeSubscription := false
for _, sub := range org.Subscriptions {
if sub.Active && sub.Amount != "0" {
hasActivePaidSubscription = true
}
if sub.Amount == "0" && sub.Reference == "" {
hasFreeSubscription = true
}
}
if hasActivePaidSubscription && hasFreeSubscription {
// Remove free subscriptions since user has active paid plan
var filteredSubs []PaymentSubscription
for _, sub := range org.Subscriptions {
if !(sub.Amount == "0" && sub.Reference == "") {
filteredSubs = append(filteredSubs, sub)
}
}
org.Subscriptions = filteredSubs
log.Printf("[INFO] Removed free subscription for org %s (active paid subscription exists)", org.Id)
} else if !hasActivePaidSubscription && !hasFreeSubscription {
// No active paid subscription and no free plan, add one
org.Subscriptions = append(org.Subscriptions, BuildBaseSubscription(*org, 2000))
log.Printf("[INFO] Added free subscription for org %s (no active paid subscriptions found)", org.Id)
}
}
if err := SetOrg(ctx, *org, org.Id); err != nil {
log.Printf("[WARNING] Failed to update subscription for org %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
return
}
sendOrgUpdaterHook := false
if len(tmpData.Image) > 0 {
org.Image = tmpData.Image
}
if len(tmpData.Name) > 0 {
org.Name = tmpData.Name
}
if len(tmpData.Description) > 0 {
org.Description = tmpData.Description
}
if len(tmpData.Defaults.AppDownloadRepo) > 0 || len(tmpData.Defaults.AppDownloadBranch) > 0 || len(tmpData.Defaults.WorkflowDownloadRepo) > 0 || len(tmpData.Defaults.WorkflowDownloadBranch) > 0 || len(tmpData.Defaults.NotificationWorkflow) > 0 || len(tmpData.Defaults.DocumentationReference) > 0 {
org.Defaults = tmpData.Defaults
}
if len(tmpData.CompanyType) > 0 {
org.CompanyType = tmpData.CompanyType
if len(org.CompanyType) == 0 {
sendOrgUpdaterHook = true
}
}
if len(tmpData.Tutorial) > 0 {
if tmpData.Tutorial == "welcome" {
sendOrgUpdaterHook = true
}
}
/*
// Old code that had frontend buttons.
// Now we discover this instead
if len(tmpData.Priority) > 0 {
if len(org.MainPriority) == 0 {
org.MainPriority = tmpData.Priority
sendOrgUpdaterHook = true
}
found := false
for _, prio := range org.Priorities {
if prio.Name == tmpData.Priority {
found = true
}
}
if !found {
org.Priorities = append(org.Priorities, Priority{
Name: tmpData.Priority,
Description: fmt.Sprintf("Priority %s decided by user.", tmpData.Priority),
Type: "usecases",
Active: true,
URL: fmt.Sprintf("/usecases"),
})
}
}
*/
// Update Billing email alert threshold
tmpDataAlert := tmpData.Billing.AlertThreshold
orgAlertThreshold := org.Billing.AlertThreshold
if len(tmpDataAlert) > 0 {
if len(tmpDataAlert) != len(orgAlertThreshold) {
org.Billing.AlertThreshold = tmpData.Billing.AlertThreshold
} else {
for i := 0; i < len(tmpDataAlert); i++ {
if tmpDataAlert[i].Percentage != orgAlertThreshold[i].Percentage || tmpDataAlert[i].Count != orgAlertThreshold[i].Count {
org.Billing.AlertThreshold = tmpData.Billing.AlertThreshold
break
}
}
}
}
if tmpData.Editing == "app_runs_hard_limit" && tmpData.Billing.AppRunsHardLimit != org.Billing.AppRunsHardLimit {
org.Billing.AppRunsHardLimit = tmpData.Billing.AppRunsHardLimit
}
if user.SupportAccess && tmpData.Editing == "internal_appruns_hard_limit" && tmpData.Billing.InternalAppRunsHardLimit != org.Billing.InternalAppRunsHardLimit {
org.Billing.InternalAppRunsHardLimit = tmpData.Billing.InternalAppRunsHardLimit
}
//Update mfa required value
if tmpData.MFARequired != org.MFARequired {
log.Printf("[AUDIT] Setting MFA required to %t for org %s (%s)", tmpData.MFARequired, org.Name, org.Id)
org.MFARequired = tmpData.MFARequired
}
if tmpData.Editing == "sso_config" {
log.Printf("[AUDIT] Editing SSO config for org %s (%s)", org.Name, org.Id)
org.SSOConfig = tmpData.SSOConfig
}
if len(tmpData.SSOConfig.SSOCertificate) > 0 {
savedCert := fixCertificate(tmpData.SSOConfig.SSOCertificate)
certLen := len([]byte(savedCert))
log.Printf("[INFO] Stripped down cert from %d to %d", len(tmpData.SSOConfig.SSOCertificate), len(savedCert))
// If the savedCert value is more than 1500 bytes in length then store it in SSOLongCertificate
if certLen > 1500 {
log.Printf("[INFO] Storing long sso certificate (>1500 bytes) for org %s (%s)", org.Name, org.Id)
org.SSOConfig.SSOLongCertificate = savedCert
org.SSOConfig.SSOCertificateHash = ssoCertHash(savedCert)
org.SSOConfig.SSOCertificate = ""
} else {
log.Printf("[INFO] Storing short sso certificate (<1500 bytes) for org %s (%s)", org.Name, org.Id)
org.SSOConfig.SSOCertificate = savedCert
org.SSOConfig.SSOCertificateHash = ""
org.SSOConfig.SSOLongCertificate = ""
}
}
if len(org.Defaults.NotificationWorkflow) > 0 && len(org.Defaults.NotificationWorkflow) != 36 {
log.Printf("[WARNING] Notification Workflow ID %s is not valid.", org.Defaults.NotificationWorkflow)
}
if len(tmpData.LeadInfo) > 0 && user.SupportAccess {
//log.Printf("[INFO] Updating lead info for %s to %s", org.Id, tmpData.LeadInfo)
// Make a new one, as to start with all from false
newLeadinfo := LeadInfo{}
for _, lead := range tmpData.LeadInfo {
if lead == "testing shuffle" || lead == "testing_shuffle" {
newLeadinfo.TestingShuffle = true
}
if lead == "contacted" {
newLeadinfo.Contacted = true
}
if lead == "student" {
newLeadinfo.Student = true
}
if lead == "lead" {
newLeadinfo.Lead = true
}
if lead == "pov" {
newLeadinfo.POV = true
}
if lead == "demo started" {
newLeadinfo.DemoDone = true
}
if lead == "customer" {
newLeadinfo.Customer = true
}
if lead == "old lead" {
newLeadinfo.OldLead = true
}
if lead == "old customer" {
newLeadinfo.OldCustomer = true
}
if lead == "opensource" || lead == "open source" {
newLeadinfo.OpenSource = true
}
if lead == "internal" {
newLeadinfo.Internal = true
}
if lead == "creator" {
newLeadinfo.Creator = true
}
if lead == "tech partner" {
newLeadinfo.TechPartner = true
}
if lead == "integration partner" {
newLeadinfo.IntegrationPartner = true
}
if lead == "distribution partner" {
newLeadinfo.DistributionPartner = true
}
if lead == "service partner" {
newLeadinfo.ServicePartner = true
}
if lead == "channel partner" {
newLeadinfo.ChannelPartner = true
}
}
org.LeadInfo = newLeadinfo
// Check for ORG_CHANGE_WEBHOOK
orgWebhook := os.Getenv("ORG_CHANGE_WEBHOOK")
if orgWebhook != "" && strings.HasPrefix(orgWebhook, "http") {
// Make a copy of org to be modified without modifying the original
tmpOrg := *org
tmpOrg.Users = []User{}
tmpOrg.Subscriptions = []PaymentSubscription{}
tmpOrg.Image = ""
tmpOrg.ActiveApps = []string{}
tmpOrg.SyncUsage = SyncUsage{}
tmpOrg.SSOConfig = SSOConfig{}
tmpOrg.SecurityFramework = Categories{}
tmpOrg.Priorities = []Priority{}
tmpOrg.Interests = []Priority{}
tmpOrg.OrgAuth = OrgAuth{}
tmpOrg.Billing = Billing{}
mappedData, err := json.Marshal(tmpOrg)
if err != nil {
log.Printf("[WARNING] Marshal error for org sending: %s", err)
} else {
req, err := http.NewRequest(
"POST",
orgWebhook,
bytes.NewBuffer(mappedData),
)
client := &http.Client{
Timeout: 3 * time.Second,
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Failed request to signup webhook FOR ORG (2): %s", err)
} else {
defer res.Body.Close()
log.Printf("[INFO] Successfully ran org priority webhook")
}
}
}
}
if len(tmpData.CreatorConfig) > 0 {
// Check if they're a creator already
if tmpData.CreatorConfig == "join" {
if org.CreatorId != "" {
log.Printf("[WARNING] Org %s is already a creator", org.Id)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Make md5 from current ID (to make it replicable)
hasher := md5.New()
hasher.Write([]byte(org.Id))
creatorId := hex.EncodeToString(hasher.Sum(nil))
log.Printf("[INFO] Org %s (%s) is joining creators with ID %s", org.Name, org.Id, creatorId)
org.CreatorId = creatorId
parsedCreatorUser := User{
Id: creatorId,
Username: org.Name,
}
parsedCreatorUser.PublicProfile.GithubAvatar = org.Image
parsedCreatorUser.PublicProfile.GithubUsername = org.Name
HandleAlgoliaCreatorUpload(ctx, parsedCreatorUser, false, true)
// Should create a new user with the same ID as the org creatorId
// This is to save public information about the org, which is used for verifying access in all other APIs
// In short: It's a way to NOT have to make all old Creator API's also support orgs. A hack, but it works
// Try to get the user
foundUser, err := GetUser(ctx, creatorId)
if err != nil {
log.Printf("[WARNING] Failed to get creator user %s: %s", creatorId, err)
// Create the user
creatorUser := parsedCreatorUser
creatorUser.PublicProfile.Public = true
creatorUser.PublicProfile.GithubUserid = org.CreatorId
creatorUser.PublicProfile.GithubUsername = org.Name
creatorUser.PublicProfile.GithubAvatar = org.Image
SetUser(ctx, &creatorUser, false)
} else {
log.Printf("[INFO] Creator user %s already exists. Should set back to public.", creatorId)
foundUser.PublicProfile.Public = true
foundUser.PublicProfile.GithubUsername = org.Name
SetUser(ctx, foundUser, false)
}
org.LeadInfo.Creator = true
} else if tmpData.CreatorConfig == "leave" {
log.Printf("[INFO] Org %s is leaving creators", org.Id)
if org.CreatorId != "" {
// Remove item with the ID from Algolia
foundUser, err := GetUser(ctx, org.CreatorId)
if err == nil {
foundUser.PublicProfile.Public = false
SetUser(ctx, foundUser, false)
}
err = HandleAlgoliaCreatorDeletion(ctx, org.CreatorId)
if err != nil {
log.Printf("[WARNING] Failed to remove creator %s (%s) from Algolia: %s", org.Name, org.CreatorId, err)
} else {
org.CreatorId = ""
}
}
org.LeadInfo.Creator = false
}
}
// if tmpData.Subscription.EulaSigned == true {
// log.Printf("[DEBUG] EULA signed for %s", org.Id)
// // Compare cloud vs onprem
// sigSample := getSignatureSample(*org)
// if len(sigSample.Eula) > 0 && sigSample.Eula == tmpData.Subscription.Eula && sigSample.Name == tmpData.Subscription.Name && sigSample.Active {
// for subIndex, sub := range org.Subscriptions {
// if len(sub.EulaSignedBy) == 0 {
// org.Subscriptions[subIndex].EulaSignedBy = user.Username
// }
// }
// org.Subscriptions = append(org.Subscriptions, tmpData.Subscription)
// org.EulaSignedBy = user.Username
// org.EulaSigned = true
// }
// }
if project.Environment == "cloud" && user.SupportAccess && tmpData.SyncFeatures.Editing {
log.Printf("[DEBUG] Updating features for org %s (%s)", org.Name, org.Id)
org.SyncFeatures = tmpData.SyncFeatures
org.SyncFeatures.Editing = false
}
if tmpData.EditingBranding {
log.Printf("[DEBUG] Updating branding for org %s (%s)", org.Name, org.Id)
org.Branding = tmpData.Branding
}
// check if user is editing sync features of suborg from parent org
if project.Environment == "cloud" && !user.SupportAccess && tmpData.SyncFeatures.Editing && tmpData.Editing != "app_runs_hard_limit" {
log.Printf("[WARNING] User %s (%s) is trying to edit sync features of suborg %s (%s)", user.Username, user.Id, org.Name, org.Id)
// check whether user org id is suborg of parent org
parentOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed to get parent org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// loop through all child orgs to check if suborg id is present in child orgs
found := false
for _, childOrg := range parentOrg.ChildOrgs {
if childOrg.Id == org.Id {
found = true
break
}
}
if !found {
log.Printf("[WARNING] User %s (%s) is trying to edit sync features of suborg %s (%s) but is not allowed (1)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// if parent org app execution limit is <= 10k then user can not edit sync features of suborg
if parentOrg.SyncFeatures.AppExecutions.Limit <= 10000 {
log.Printf("[WARNING] User %s (%s) is trying to edit sync features of suborg %s (%s) but is not allowed (2)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// User can't assign more app execution and workfflow runs than parent org to suborg
// check if current org app execution limit is changed with tmpData org app execution limit
if tmpData.SyncFeatures.AppExecutions.Limit >= parentOrg.SyncFeatures.AppExecutions.Limit && tmpData.SyncFeatures.AppExecutions.Limit != org.SyncFeatures.AppExecutions.Limit {
log.Printf("[WARNING] User %s (%s) is trying to edit sync features of suborg %s (%s) but is not allowed (3)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpData.SyncFeatures.WorkflowExecutions.Limit >= parentOrg.SyncFeatures.WorkflowExecutions.Limit && tmpData.SyncFeatures.WorkflowExecutions.Limit != org.SyncFeatures.WorkflowExecutions.Limit {
log.Printf("[WARNING] User %s (%s) is trying to edit sync features of suborg %s (%s) but is not allowed (4)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[DEBUG] User %s (%s) is allowed to edit sync features of suborg %s (%s)", user.Username, user.Id, org.Name, org.Id)
org.SyncFeatures = tmpData.SyncFeatures
org.SyncFeatures.Editing = false
}
if (len(tmpData.Billing.Consultation.Hours) > 0 || len(tmpData.Billing.Consultation.Minutes) > 0) && user.SupportAccess {
org.Billing.Consultation = tmpData.Billing.Consultation
}
// Built a system around this now, which checks for the actual org.
// if requestdata.Environment == "cloud" && project.Environment != "cloud" {
//if project.Environment != "cloud" && len(org.SSOConfig.SSOEntrypoint) > 0 && len(org.ManagerOrgs) == 0 {
// //log.Printf("[INFO] Should set SSO entrypoint to %s", org.SSOConfig.SSOEntrypoint)
// SSOUrl = org.SSOConfig.SSOEntrypoint
//}
log.Printf("[DEBUG] Updating org %s (%s) with %d users", org.Name, org.Id, len(org.Users))
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[ERROR] Failed to edit org %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
// Sends tracker for this on cloud
if sendOrgUpdaterHook && project.Environment == "cloud" {
signupWebhook := os.Getenv("WEBSITE_ORG_WEBHOOK")
if strings.HasPrefix(signupWebhook, "http") {
curIndex := -1
for orgIndex, orguser := range org.Users {
if orguser.Id == user.Id {
curIndex = orgIndex
}
}
if curIndex >= 0 {
user.Password = ""
user.Session = ""
user.ApiKey = ""
user.LoginInfo = []LoginInfo{}
user.PrivateApps = []WorkflowApp{}
org.Users[curIndex] = user
}
mappedData, err := json.Marshal(org)
if err != nil {
log.Printf("[WARNING] Marshal error for org sending: %s", err)
} else {
req, err := http.NewRequest(
"POST",
signupWebhook,
bytes.NewBuffer(mappedData),
)
client := &http.Client{
Timeout: 3 * time.Second,
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Failed request to signup webhook FOR ORG (2): %s", err)
} else {
log.Printf("[INFO] Successfully ran org priority webhook")
}
defer res.Body.Close()
}
}
}
GetTutorials(ctx, *org, true)
ip := GetRequestIp(request)
log.Printf("[AUDIT] Org %s (%s) updated by user %s (%s) from IP %s - priorities: %d", org.Name, org.Id, user.Username, user.Id, ip, len(org.Priorities))
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Successfully updated org"}`)))
}
func sendMailSendgrid(toEmail []string, subject, body string, emailApp bool, BccAddresses []string) error {
log.Printf("[DEBUG] In mail sending with subject %s and body length %s. TO: %s", subject, body, toEmail)
srequest := sendgrid.GetRequest(os.Getenv("SENDGRID_API_KEY"), "/v3/mail/send", "https://api.sendgrid.com")
srequest.Method = "POST"
type SendgridContent struct {
Type string `json:"type"`
Value string `json:"value"`
}
type SendgridEmail struct {
Email string `json:"email"`
}
type SendgridPersonalization struct {
To []SendgridEmail `json:"to"`
Bcc []SendgridEmail `json:"bcc"`
Subject string `json:"subject"`
}
type sendgridMailBody struct {
Personalizations []SendgridPersonalization `json:"personalizations"`
From SendgridEmail `json:"from"`
Content []SendgridContent `json:"content"`
}
body = strings.Replace(body, "\n", "
", -1)
newBody := sendgridMailBody{
Personalizations: []SendgridPersonalization{
{
To: []SendgridEmail{},
Subject: subject,
},
},
From: SendgridEmail{
Email: "Shuffle Support ",
},
Content: []SendgridContent{
{
Type: "text/html",
Value: body,
},
},
}
if emailApp {
newBody.From.Email = "Shuffle Email App "
}
for _, email := range toEmail {
newBody.Personalizations[0].To = append(newBody.Personalizations[0].To,
SendgridEmail{
Email: strings.TrimSpace(email),
})
}
// Conditionally add BCC addresses if they exist
if len(BccAddresses) > 0 {
for _, bccEmail := range BccAddresses {
newBody.Personalizations[0].Bcc = append(newBody.Personalizations[0].Bcc,
SendgridEmail{
Email: strings.TrimSpace(bccEmail),
})
}
}
parsedBody, err := json.Marshal(newBody)
if err != nil {
log.Printf("[ERROR] Failed to parse JSON in sendmail: %s", err)
return err
}
srequest.Body = parsedBody
log.Printf("[DEBUG] Email: %s\n\n", srequest.Body)
response, err := sendgrid.API(srequest)
if err != nil {
log.Println(err)
} else {
if response.StatusCode >= 300 {
log.Printf("[DEBUG] Failed sending mail! Statuscode: %d. Body: %s", response.StatusCode, response.Body)
} else {
log.Printf("[DEBUG] Successfully sent email! Statuscode: %d. Body: %s", response.StatusCode, response.Body)
}
return nil
//log.Printf(response.Headers)
}
return err
}
func sendMailSendgridV2(toEmail []string, subject string, substitutions map[string]interface{}, emailApp bool, templateID string, bccEmail []string) error {
log.Printf("[DEBUG] In mail sending with subject %s. TO: %s, BCC: %s", subject, toEmail, bccEmail)
srequest := sendgrid.GetRequest(os.Getenv("SENDGRID_API_KEY"), "/v3/mail/send", "https://api.sendgrid.com")
srequest.Method = "POST"
type SendgridEmail struct {
Email string `json:"email"`
}
type SendgridPersonalization struct {
To []SendgridEmail `json:"to"`
Bcc []SendgridEmail `json:"bcc,omitempty"`
Subject string `json:"subject"`
DynamicTemplateData map[string]interface{} `json:"dynamic_template_data,omitempty"`
}
type sendgridMailBody struct {
Personalizations []SendgridPersonalization `json:"personalizations"`
From SendgridEmail `json:"from"`
TemplateID string `json:"template_id"`
}
newBody := sendgridMailBody{
Personalizations: []SendgridPersonalization{
{
To: []SendgridEmail{},
Bcc: []SendgridEmail{},
Subject: subject,
DynamicTemplateData: substitutions,
},
},
From: SendgridEmail{
Email: "Shuffle Support ",
},
TemplateID: templateID,
}
if emailApp {
newBody.From.Email = "Shuffle Email App "
}
for _, email := range toEmail {
newBody.Personalizations[0].To = append(newBody.Personalizations[0].To,
SendgridEmail{
Email: strings.TrimSpace(email),
})
}
for _, email := range bccEmail {
newBody.Personalizations[0].Bcc = append(newBody.Personalizations[0].Bcc,
SendgridEmail{
Email: strings.TrimSpace(email),
})
}
parsedBody, err := json.Marshal(newBody)
if err != nil {
log.Printf("[ERROR] Failed to parse JSON in sendmail: %s", err)
return err
}
srequest.Body = parsedBody
response, err := sendgrid.API(srequest)
if err != nil {
log.Println(err)
} else {
if response.StatusCode >= 300 {
log.Printf("[DEBUG] Failed sending mail! Statuscode: %d. Body: %s", response.StatusCode, response.Body)
} else {
log.Printf("[DEBUG] Successfully sent email! Statuscode: %d. Body: %s", response.StatusCode, response.Body)
}
return nil
}
return err
}
func CheckWorkflowApp(workflowApp WorkflowApp) error {
// Validate fields
if workflowApp.Name == "" {
return errors.New("App field name doesn't exist")
}
if workflowApp.Description == "" {
return errors.New("App field description doesn't exist")
}
if workflowApp.AppVersion == "" {
return errors.New("App field app_version doesn't exist")
}
if workflowApp.ContactInfo.Name == "" {
return errors.New("App field contact_info.name doesn't exist")
}
return nil
}
func AbortExecution(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID to abort is not valid"}`))
return
}
executionId := location[6]
if len(executionId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "ExecutionID not valid"}`))
return
}
ctx := GetContext(request)
workflowExecution, err := GetWorkflowExecution(ctx, executionId)
if err != nil {
log.Printf("[ERROR][%s] Failed getting execution (abort): %s", executionId, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting execution ID '%s' because it doesn't exist (abort)."}`, executionId)))
return
}
apikey := request.Header.Get("Authorization")
parsedKey := ""
if strings.HasPrefix(apikey, "Bearer ") {
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) == 2 {
parsedKey = apikeyCheck[1]
}
}
// Checks the users' role and such if the key fails
//log.Printf("Abort info: %s vs %s", workflowExecution.Authorization, parsedKey)
if workflowExecution.Authorization != parsedKey {
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in abort workflow: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Id != workflowExecution.Workflow.Owner {
if workflowExecution.Workflow.OrgId == user.ActiveOrg.Id && user.Role == "admin" {
log.Printf("[AUDIT] User %s is aborting execution %s as admin", user.Username, workflowExecution.Workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for ABORT of workflowexecution workflow %s", user.Username, workflowExecution.Workflow.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
} else {
//log.Printf("[INFO] API key to abort/finish execution %s is correct.", executionId)
}
if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" || workflowExecution.Status == "FINISHED" {
//err = SetWorkflowExecution(ctx, *workflowExecution, true)
//if err != nil {
//}
log.Printf("[INFO] Stopped execution of %s with status %s", executionId, workflowExecution.Status)
if len(workflowExecution.ExecutionParent) > 0 {
}
//ExecutionSource string `json:"execution_source" datastore:"execution_source"`
//ExecutionParent string `json:"execution_parent" datastore:"execution_parent"`
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Status for %s is %s, which can't be aborted."}`, executionId, workflowExecution.Status)))
return
}
topic := "workflowexecution"
workflowExecution.CompletedAt = int64(time.Now().Unix())
workflowExecution.Status = "ABORTED"
log.Printf("[INFO] Running shutdown (abort) of execution %s", workflowExecution.ExecutionId)
lastResult := ""
newResults := []ActionResult{}
// type ActionResult struct {
for _, result := range workflowExecution.Results {
if result.Status == "EXECUTING" {
result.Status = "ABORTED"
result.Result = "Aborted because of error in another node (1)"
}
if len(result.Result) > 0 && result.Status == "SUCCESS" {
lastResult = result.Result
}
newResults = append(newResults, result)
}
workflowExecution.Results = newResults
if len(workflowExecution.Result) == 0 {
workflowExecution.Result = lastResult
}
addResult := true
for _, result := range workflowExecution.Results {
if result.Status != "SKIPPED" {
addResult = false
}
}
extra := 0
for _, trigger := range workflowExecution.Workflow.Triggers {
//log.Printf("Appname trigger (0): %s", trigger.AppName)
if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
extra += 1
}
}
parsedReason := "An error occurred during execution of this node. This may be due to the Workflow being Aborted, or an error in the node itself."
reason, reasonok := request.URL.Query()["reason"]
if reasonok {
parsedReason = reason[0]
// Custom reason handler for weird inputs
if strings.Contains(parsedReason, "manifest for registry") {
foundImageSplit := strings.Split(parsedReason, " ")
foundImage := ""
if len(foundImageSplit) > 7 {
foundImage = foundImageSplit[6]
foundImageSplit = strings.Split(foundImageSplit[6], ":")
if len(foundImageSplit) > 1 {
foundImage = foundImageSplit[1]
}
}
parsedReason = fmt.Sprintf("Couldn't find the Docker image %s. Did you activate the app?", foundImage)
}
}
returnData := SubflowData{
Success: false,
Result: parsedReason,
}
reasonData, err := json.Marshal(returnData)
if err != nil {
reasonData = []byte(parsedReason)
}
if len(workflowExecution.Results) == 0 || addResult {
newaction := Action{
ID: workflowExecution.Start,
}
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == workflowExecution.Start {
newaction = action
break
}
}
workflowExecution.Results = append(workflowExecution.Results, ActionResult{
Action: newaction,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: string(reasonData),
StartedAt: workflowExecution.StartedAt,
CompletedAt: workflowExecution.StartedAt,
Status: "FAILURE",
})
} else if len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra {
log.Printf("[INFO] DONE - Nothing to add during abort!")
} else {
//log.Printf("VALIDATING INPUT!")
node, nodeok := request.URL.Query()["node"]
if nodeok {
nodeId := node[0]
log.Printf("[INFO] Found abort node %s", nodeId)
newaction := Action{
ID: nodeId,
}
// Check if result exists first
found := false
for _, result := range workflowExecution.Results {
if result.Action.ID == nodeId {
found = true
break
}
}
if !found {
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == nodeId {
newaction = action
break
}
}
workflowExecution.Results = append(workflowExecution.Results, ActionResult{
Action: newaction,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: string(reasonData),
StartedAt: workflowExecution.StartedAt,
CompletedAt: workflowExecution.StartedAt,
Status: "FAILURE",
})
}
}
}
for resultIndex, result := range workflowExecution.Results {
for parameterIndex, param := range result.Action.Parameters {
if param.Configuration {
workflowExecution.Results[resultIndex].Action.Parameters[parameterIndex].Value = ""
}
}
}
for actionIndex, action := range workflowExecution.Workflow.Actions {
for parameterIndex, param := range action.Parameters {
if param.Configuration {
//log.Printf("Cleaning up %s in %s", param.Name, action.Name)
workflowExecution.Workflow.Actions[actionIndex].Parameters[parameterIndex].Value = ""
}
}
}
// This is the same as aborted
IncrementCache(ctx, workflowExecution.ExecutionOrg, "workflow_executions_failed")
err = SetWorkflowExecution(ctx, *workflowExecution, true)
if err != nil {
log.Printf("[WARNING] Error saving workflow execution for updates when aborting (2) %s: %s", topic, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution status to abort"}`)))
return
} else {
log.Printf("[INFO][%s] Set workflowexecution to aborted.", workflowExecution.ExecutionId)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func SanitizeWorkflow(workflow Workflow) Workflow {
for _, trigger := range workflow.Triggers {
_ = trigger
}
for _, action := range workflow.Actions {
_ = action
}
for _, variable := range workflow.WorkflowVariables {
_ = variable
}
workflow.Org = []OrgMini{}
workflow.OrgId = ""
workflow.ExecutingOrg = OrgMini{}
workflow.PreviouslySaved = false
// Add Gitguardian or similar secrets discovery
return workflow
}
// Starts a new webhook
func HandleNewHook(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in set new hook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to make new hook: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
type requestData struct {
Type string `json:"type"`
Description string `json:"description"`
Id string `json:"id"`
Name string `json:"name"`
Workflow string `json:"workflow"`
Start string `json:"start"`
Environment string `json:"environment"`
Auth string `json:"auth"`
CustomResponse string `json:"custom_response"`
Version string `json:"version" datastore:"version"`
VersionTimeout int `json:"version_timeout" datastore:"version_timeout"`
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Body data error in webhook set: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
var requestdata requestData
err = json.Unmarshal([]byte(body), &requestdata)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling inputdata for webhook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newId := requestdata.Id
if len(newId) != 36 {
log.Printf("[WARNING] Bad webhook ID: %s", newId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid Webhook ID: bad formatting"}`))
return
}
if requestdata.Id == "" || requestdata.Name == "" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Required fields id and name can't be empty"}`))
return
}
validTypes := []string{
"webhook",
}
isTypeValid := false
for _, thistype := range validTypes {
if requestdata.Type == thistype {
isTypeValid = true
break
}
}
if !(isTypeValid) {
log.Printf("[WARNING] Type %s is not valid. Try any of these: %s", requestdata.Type, strings.Join(validTypes, ", "))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
originalWorkflow, err := GetWorkflow(ctx, requestdata.Workflow)
if err != nil {
log.Printf("[WARNING] Failed getting workflow %s: %s", requestdata.Workflow, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow doesn't exist"}`))
return
}
originalHook, err := GetHook(ctx, newId)
if err == nil {
log.Printf("[WARNING] Hook with ID %s doesn't exist", newId)
}
if originalWorkflow.OrgId != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s doesn't have access to workflow %s", user.Username, requestdata.Workflow)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if (!(user.SupportAccess || user.Id == originalHook.Owner) || len(user.Id) == 0) && originalHook.Id != "" {
if originalHook.OrgId != user.ActiveOrg.Id && originalHook.OrgId != "" {
log.Printf("[WARNING] User %s doesn't have access to hook %s", user.Username, originalHook.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User doesn't have access to hook"}`))
return
}
}
// Let remote endpoint handle access checks (shuffler.io)
baseUrl := "https://shuffler.io"
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
baseUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
currentUrl := fmt.Sprintf("%s/api/v1/hooks/webhook_%s", baseUrl, newId)
startNode := requestdata.Start
if requestdata.Environment == "cloud" && project.Environment != "cloud" {
// https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c
log.Printf("[INFO] Should START a cloud webhook for url %s for startnode %s", currentUrl, startNode)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed finding org %s: %s", org.Id, err)
return
}
action := CloudSyncJob{
Type: "webhook",
Action: "start",
OrgId: org.Id,
PrimaryItemId: newId,
SecondaryItem: startNode,
ThirdItem: requestdata.Workflow,
FourthItem: requestdata.Auth,
}
err = executeCloudAction(action, org.SyncConfig.Apikey)
if err != nil {
log.Printf("[WARNING] Failed cloud action START webhook execution: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
} else {
log.Printf("[INFO] Successfully set up cloud action schedule")
}
}
hook := Hook{
Id: newId,
Start: startNode,
Workflows: []string{requestdata.Workflow},
Info: Info{
Name: requestdata.Name,
Description: requestdata.Description,
Url: fmt.Sprintf("%s/api/v1/hooks/webhook_%s", baseUrl, newId),
},
Type: "webhook",
Owner: user.Username,
Status: "uninitialized",
Actions: []HookAction{
HookAction{
Type: "workflow",
Name: requestdata.Name,
Id: requestdata.Workflow,
Field: "",
},
},
Running: false,
OrgId: user.ActiveOrg.Id,
Environment: requestdata.Environment,
Auth: requestdata.Auth,
CustomResponse: requestdata.CustomResponse,
Version: requestdata.Version,
VersionTimeout: requestdata.VersionTimeout,
}
hook.Status = "running"
hook.Running = true
err = SetHook(ctx, hook)
if err != nil {
log.Printf("[WARNING] Failed setting hook: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
// set the same for the workflow
workflow, err := GetWorkflow(ctx, requestdata.Workflow)
if err != nil {
log.Printf("[WARNING] Failed getting workflow %s: %s", requestdata.Workflow, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// get the webhook trigger with the same id
for triggerIndex, trigger := range workflow.Triggers {
if trigger.ID == newId {
workflow.Triggers[triggerIndex].Status = "running"
log.Printf("[INFO] Changed status of trigger %s to running", newId)
break
}
}
// update the workflow
err = SetWorkflow(ctx, *workflow, workflow.ID)
if err != nil {
log.Printf("[WARNING] Failed setting workflow %s: %s", workflow.ID, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] Set up a new hook with ID %s and environment %s", newId, hook.Environment)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func HandleDeleteHook(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in delete hook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to delete hook: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
// Check if fileId has the prefix "webhook_"
if strings.HasPrefix(fileId, "webhook_") {
fileId = strings.TrimPrefix(fileId, "webhook_")
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when deleting hook is not valid"}`))
return
}
ctx := GetContext(request)
hook, err := GetHook(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Failed getting hook %s (delete): %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
//if user.Id != hook.Owner && user.ActiveOrg.Id != hook.OrgId {
// log.Printf("[WARNING] Wrong user (%s) for workflow %s", user.Username, hook.Id)
// resp.WriteHeader(401)
// resp.Write([]byte(`{"success": false}`))
// return
//}
if user.Id != hook.Owner || len(user.Id) == 0 {
if hook.OrgId == user.ActiveOrg.Id && user.Role == "admin" {
log.Printf("[AUDIT] User %s is stopping hook for workflow %s as admin. Owner: %s", user.Username, hook.Workflows[0], hook.Owner)
} else {
log.Printf("[AUDIT] Wrong user (%s) for hook %s (stop hook)", user.Username, hook.Workflows[0])
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if len(hook.Workflows) > 0 {
//err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1, user.ActiveOrg.Id)
//if err != nil {
// log.Printf("Failed to increase total workflows: %s", err)
//}
}
hook.Status = "stopped"
err = SetHook(ctx, *hook)
if err != nil {
log.Printf("[WARNING] Failed setting hook: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// find workflow and set status to stopped
workflow, err := GetWorkflow(ctx, hook.Workflows[0])
if err != nil {
log.Printf("[WARNING] Failed getting workflow %s: %s", hook.Workflows[0], err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if len(workflow.Triggers) > 0 {
for triggerIndex, trigger := range workflow.Triggers {
if trigger.ID == fileId {
workflow.Triggers[triggerIndex].Status = "stopped"
break
}
}
}
if hook.Environment == "cloud" && project.Environment != "cloud" {
log.Printf("[INFO] Should STOP cloud webhook https://shuffler.io/api/v1/hooks/webhook_%s", hook.Id)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed finding org %s: %s", org.Id, err)
return
}
action := CloudSyncJob{
Type: "webhook",
Action: "stop",
OrgId: org.Id,
PrimaryItemId: hook.Id,
}
if len(hook.Workflows) > 0 {
action.SecondaryItem = hook.Workflows[0]
}
err = executeCloudAction(action, org.SyncConfig.Apikey)
if err != nil {
log.Printf("Failed cloud action STOP execution: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
// https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c
}
err = DeleteKey(ctx, "hooks", fileId)
if err != nil {
log.Printf("[WARNING] Error deleting hook %s for %s: %s", fileId, user.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed deleting the hook."}`))
return
}
log.Printf("[INFO] Successfully deleted webhook %s", fileId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "Stopped webhook"}`))
}
// startWebhookTrigger - Core function to start a webhook trigger without HTTP handlers
func startWebhookTrigger(ctx context.Context, workflowId, triggerId, triggerName, startNode, environment, auth, customResponse, version string, versionTimeout int, user User, orgId string) error {
// Validate webhook ID length (exactly like HandleNewHook)
if len(triggerId) != 36 {
return fmt.Errorf("invalid Webhook ID: bad formatting - must be 36 characters, got %d", len(triggerId))
}
// Get the workflow first
workflow, err := GetWorkflow(ctx, workflowId)
if err != nil {
return fmt.Errorf("failed getting workflow %s: %s", workflowId, err)
}
// Check if hook already exists
originalHook, err := GetHook(ctx, triggerId)
if err == nil {
log.Printf("[WARNING] Hook with ID %s already exists", triggerId)
}
// Validate workflow access (exactly like HandleNewHook)
if workflow.OrgId != user.ActiveOrg.Id {
return fmt.Errorf("user %s doesn't have access to workflow %s", user.Username, workflowId)
}
// Validate hook access if it exists (exactly like HandleNewHook)
if (!(user.SupportAccess || user.Id == originalHook.Owner) || len(user.Id) == 0) && originalHook.Id != "" {
if originalHook.OrgId != user.ActiveOrg.Id && originalHook.OrgId != "" {
return fmt.Errorf("user %s doesn't have access to hook %s", user.Username, originalHook.Id)
}
}
// Generate webhook URL (exactly like HandleNewHook)
baseUrl := "https://shuffler.io"
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
baseUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
webhookUrl := fmt.Sprintf("%s/api/v1/hooks/webhook_%s", baseUrl, triggerId)
// Handle cloud webhook setup if needed (exactly like HandleNewHook)
if environment == "cloud" && project.Environment != "cloud" {
log.Printf("[INFO] Should START a cloud webhook for url %s for startnode %s", webhookUrl, startNode)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
return fmt.Errorf("failed finding org %s: %s", user.ActiveOrg.Id, err)
}
action := CloudSyncJob{
Type: "webhook",
Action: "start",
OrgId: org.Id,
PrimaryItemId: triggerId,
SecondaryItem: startNode,
ThirdItem: workflowId,
FourthItem: auth,
}
err = executeCloudAction(action, org.SyncConfig.Apikey)
if err != nil {
return fmt.Errorf("failed cloud action START webhook execution: %s", err)
} else {
log.Printf("[INFO] Successfully set up cloud action schedule")
}
}
// Create the hook object (EXACTLY like HandleNewHook)
hook := Hook{
Id: triggerId,
Start: startNode,
Workflows: []string{workflowId},
Info: Info{
Name: triggerName,
Description: triggerName, // Use triggerName for description like original
Url: webhookUrl,
},
Type: "webhook",
Owner: user.Username,
Status: "uninitialized", // IMPORTANT: Start as uninitialized like original
Actions: []HookAction{
{
Type: "workflow",
Name: triggerName,
Id: workflowId,
Field: "",
},
},
Running: false, // IMPORTANT: Start as false like original
OrgId: user.ActiveOrg.Id,
Environment: environment,
Auth: auth,
CustomResponse: customResponse,
Version: version, // MISSING FIELD ADDED
VersionTimeout: versionTimeout, // MISSING FIELD ADDED
}
// EXACTLY like HandleNewHook - set to running AFTER creation
hook.Status = "running"
hook.Running = true
err = SetHook(ctx, hook)
if err != nil {
return fmt.Errorf("failed setting hook: %s", err)
}
// set the same for the workflow (exactly like HandleNewHook)
workflow, err = GetWorkflow(ctx, workflowId)
if err != nil {
return fmt.Errorf("failed getting workflow %s: %s", workflowId, err)
}
// get the webhook trigger with the same id (exactly like HandleNewHook)
for triggerIndex, trigger := range workflow.Triggers {
if trigger.ID == triggerId {
workflow.Triggers[triggerIndex].Status = "running"
log.Printf("[INFO] Changed status of trigger %s to running", triggerId)
break
}
}
// update the workflow (exactly like HandleNewHook)
err = SetWorkflow(ctx, *workflow, workflow.ID)
if err != nil {
return fmt.Errorf("failed setting workflow %s: %s", workflow.ID, err)
}
log.Printf("[INFO] Set up a new hook with ID %s and environment %s", triggerId, hook.Environment)
return nil
}
// startAllWorkflowTriggers - Start all triggers for a workflow (most useful function)
func startAllWorkflowTriggers(ctx context.Context, workflowId string, user User, orgId string) error {
workflow, err := GetWorkflow(ctx, workflowId)
if err != nil {
return fmt.Errorf("failed getting workflow %s: %s", workflowId, err)
}
if workflow.OrgId != orgId {
return fmt.Errorf("user doesn't have access to workflow %s", workflowId)
}
for _, trigger := range workflow.Triggers {
switch trigger.TriggerType {
case "WEBHOOK":
// Extract parameters
auth := ""
customResponse := ""
version := ""
versionTimeout := 0
for _, param := range trigger.Parameters {
if param.Name == "auth_headers" {
auth = param.Value
} else if param.Name == "custom_response_body" {
customResponse = param.Value
} else if param.Name == "await_response" {
version = param.Value
} else if param.Name == "version_timeout" {
if timeoutInt, err := strconv.Atoi(param.Value); err == nil {
versionTimeout = timeoutInt
}
}
}
// Find start node
startNode := workflow.Start
for _, branch := range workflow.Branches {
if branch.SourceID == trigger.ID {
startNode = branch.DestinationID
break
}
}
err := startWebhookTrigger(ctx, workflowId, trigger.ID, trigger.Label, startNode, trigger.Environment, auth, customResponse, version, versionTimeout, user, orgId)
if err != nil {
log.Printf("[ERROR] Failed starting webhook trigger %s: %s", trigger.ID, err)
return err
}
case "SCHEDULE":
err := startSchedule(trigger, user.ApiKey, *workflow)
if err != nil {
log.Printf("[ERROR] Failed starting schedule trigger %s: %s", trigger.ID, err)
return err
}
default:
log.Printf("[INFO] Trigger type %s not supported for auto-start", trigger.TriggerType)
}
}
return nil
}
func ParseVersions(versions []string) []string {
log.Printf("Versions: %s", versions)
//versions = sort.Sort(semver.Collection(versions))
return versions
}
func updateOrgAppCache(app WorkflowApp, user User) {
if len(app.ID) == 0 {
return
}
if len(user.ActiveOrg.Id) == 0 {
return
}
// Random delay from 0-2 seconds
time.Sleep(time.Duration(mathrand.Intn(1)) * time.Second)
ctx := context.Background()
cacheKey := fmt.Sprintf("apps_%s", user.ActiveOrg.Id)
cache, err := GetCache(ctx, cacheKey)
if err != nil {
//log.Printf("[WARNING] Failed getting apps for %s from cache: %s", cacheKey, err)
return
} else {
allApps := []WorkflowApp{}
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &allApps)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling cache data for apps: %s", err)
} else {
updated := false
for appIndex, thisApp := range allApps {
if thisApp.ID == app.ID {
// Skipping update for those with actions
if len(thisApp.Actions) > 1 {
return
}
updated = true
allApps[appIndex] = app
break
}
}
if !updated {
allApps = append(allApps, app)
}
cacheData, err = json.Marshal(allApps)
if err != nil {
log.Printf("[WARNING] Failed marshalling updated apps for cache: %s", err)
} else {
err = SetCache(ctx, cacheKey, cacheData, 1440)
if err != nil {
log.Printf("[WARNING] Failed updating org cache for apps: %s", err)
//log.Printf("[INFO] Updated cache for apps in org %s", user.ActiveOrg.Id)
}
}
}
}
}
func GetWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
//log.Printf("[INFO] Running GetWorkflowAppConfig for '%s'", fileId)
ctx := GetContext(request)
app, err := GetApp(ctx, fileId, User{}, false)
if err != nil {
log.Printf("[WARNING] Error getting app %s (app config): %s", fileId, err)
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// Update local stash here?
// Load config & update
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
// Must be here to not override apps
go LoadAppConfigFromMain(fileId, false)
log.Printf("[DEBUG] Redirecting App load request '%s' to main site handler (shuffler.io)", fileId)
RedirectUserRequest(resp, request)
return
}
}
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
return
}
// FIXME: Should we redirect here?
if app.Public {
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting App load request '%s' to main site handler (shuffler.io) (2)", fileId)
RedirectUserRequest(resp, request)
return
}
}
}
app.ReferenceUrl = ""
data, err := json.Marshal(app)
if err != nil {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling new parsed APP: %s"}`, err)))
return
}
appReturn := AppParser{
Success: true,
App: data,
}
appdata, err := json.Marshal(appReturn)
if err != nil {
log.Printf("[WARNING] Error parsing appReturn for app (INIT): %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling: %s"}`, err)))
return
}
user, userErr := HandleApiAuthentication(resp, request)
openapi, openapiok := request.URL.Query()["openapi"]
//if app.Sharing || app.Public || (project.Environment == "cloud" && user.Id == "what") {
//log.Printf("SHARING: %s. PUBLIC: %s", app.Sharing, app.Public)
if app.Sharing || app.Public {
if openapiok && len(openapi) > 0 && strings.ToLower(openapi[0]) == "false" {
//log.Printf("[DEBUG] Returning app '%s' without OpenAPI", fileId)
} else {
//log.Printf("CAN SHARE APP!")
parsedApi, err := GetOpenApiDatastore(ctx, fileId)
if err != nil {
go updateOrgAppCache(*app, user)
log.Printf("[WARNING] OpenApi doesn't exist for (0): %s - err: %s. Returning basic app", fileId, err)
resp.WriteHeader(200)
resp.Write(appdata)
return
}
if len(parsedApi.Body) > 0 {
if len(parsedApi.ID) > 0 {
parsedApi.Success = true
} else {
parsedApi.Success = false
}
//log.Printf("PARSEDAPI: %s", parsedApi)
openapidata, err := json.Marshal(parsedApi)
if err != nil {
log.Printf("[WARNING] Error parsing api json: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling new parsed swagger: %s"}`, err)))
return
}
appReturn.OpenAPI = openapidata
}
}
appdata, err = json.Marshal(appReturn)
if err != nil {
log.Printf("[WARNING] Error parsing appReturn for app: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling: %s"}`, err)))
return
}
go updateOrgAppCache(*app, user)
resp.WriteHeader(200)
resp.Write(appdata)
return
}
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in get app: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Modified to make it so users admins in same org can modify an app
//log.Printf("User: %s, role: %s, org: %s vs %s", user.Username, user.Role, user.ActiveOrg.Id, app.ReferenceOrg)
if project.Environment != "cloud" && len(app.ReferenceOrg) == 0 {
app.ReferenceOrg = user.ActiveOrg.Id
}
if user.Id == app.Owner || user.ActiveOrg.Id == app.ReferenceOrg || ArrayContains(app.Contributors, user.Id) {
log.Printf("[AUDIT] Got app %s (%s) with user %s (%s) in org %s", app.Name, app.ID, user.Username, user.Id, user.ActiveOrg.Id)
} else {
if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Support & Admin user %s (%s) got access to app %s (cloud only)", user.Username, user.Id, app.ID)
} else if user.Role == "admin" && app.Owner == "" {
log.Printf("[AUDIT] Any admin can GET %s (%s), since it doesn't have an owner (GET).", app.Name, app.ID)
} else {
exit := true
log.Printf("[INFO] Check published app reference ID: %#v", app.PublishedId)
if len(app.PublishedId) > 0 {
// FIXME: is this privacy / vulnerability?
// Allows parent owner to see child usage.
// Intended to allow vision of changes, and have parent app suggestions be possible
parentapp, err := GetApp(ctx, app.PublishedId, user, false)
if err == nil {
if parentapp.Owner == user.Id {
log.Printf("[AUDIT] Parent app owner %s (%s) got access to child app %s (%s)", user.Username, user.Id, app.Name, app.ID)
exit = false
//app, err := GetApp(ctx, fileId, User{}, false)
}
}
}
if exit {
log.Printf("[AUDIT] Wrong user (%s) for app %s (%s) - get app config", user.Username, app.Name, app.ID)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
}
if openapiok && len(openapi) > 0 && strings.ToLower(openapi[0]) == "false" {
//log.Printf("Should return WITHOUT openapi")
} else {
log.Printf("[INFO] Getting app %s (OpenAPI)", fileId)
parsedApi, err := GetOpenApiDatastore(ctx, fileId)
if err != nil {
log.Printf("[INFO] OpenApi doesn't exist for (1): %s - err: %s. Returning basic app.", fileId, err)
resp.WriteHeader(200)
resp.Write(appdata)
return
}
if len(parsedApi.ID) > 0 {
parsedApi.Success = true
} else {
parsedApi.Success = false
}
openapidata, err := json.Marshal(parsedApi)
if err != nil {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling new parsed swagger: %s"}`, err)))
return
}
appReturn.OpenAPI = openapidata
}
// Should add it to their cache in the background
go updateOrgAppCache(*app, user)
appdata, err = json.Marshal(appReturn)
if err != nil {
log.Printf("[WARNING] Error parsing appReturn for app: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling: %s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write(appdata)
}
func verifier() (*CodeVerifier, error) {
r := mathrand.New(mathrand.NewSource(time.Now().UnixNano()))
b := make([]byte, 32, 32)
for i := 0; i < 32; i++ {
b[i] = byte(r.Intn(255))
}
return CreateCodeVerifierFromBytes(b)
}
func GetOpenIdUrl(request *http.Request, org Org) string {
baseSSOUrl := org.SSOConfig.OpenIdAuthorization
codeChallenge := uuid.NewV4().String()
//h.Write([]byte(v.Value))
verifier, verifiererr := verifier()
if verifiererr == nil {
codeChallenge = verifier.Value
}
//log.Printf("[DEBUG] Got challenge value %s (pre state)", codeChallenge)
// https://192.168.55.222:3443/api/v1/login_openid
//location := strings.Split(request.URL.String(), "/")
//redirectUrl := url.QueryEscape("http://localhost:5001/api/v1/login_openid")
redirectUrl := url.QueryEscape(fmt.Sprintf("http://%s/api/v1/login_openid", request.Host))
if project.Environment == "cloud" {
redirectUrl = url.QueryEscape(fmt.Sprintf("https://shuffler.io/api/v1/login_openid"))
}
//Redirect url for onprem
if project.Environment != "cloud" && strings.Contains(request.Host, "shuffle-backend") && !strings.Contains(os.Getenv("BASE_URL"), "shuffle-backend") {
redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("BASE_URL")))
} else {
//check if base url exist if exist then assign the base url. This is for local testing when request.Host is is not "shuffle-backend"
if project.Environment != "cloud" && len(os.Getenv("BASE_URL")) > 0 {
redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("BASE_URL")))
} else if project.Environment != "cloud" {
//if base url not exist then assign hardcoded url for the onprem, user should not reach here but in case not set the base url hardcode it.
redirectUrl = url.QueryEscape(fmt.Sprintf("http://localhost:5001/api/v1/login_openid"))
}
}
//In any case redirect url should not be the SSO_REDIRECT_URL as it is the frontend url where user will be redirected after login.
if project.Environment != "cloud" && len(os.Getenv("SSO_REDIRECT_URL")) > 0 {
redirectUrl = url.QueryEscape(fmt.Sprintf("%s/api/v1/login_openid", os.Getenv("SSO_REDIRECT_URL")))
}
state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&challenge=%s&redirect=%s", org.Id, codeChallenge, redirectUrl)))
// has to happen after initial value is stored
if verifiererr == nil {
codeChallenge = verifier.CodeChallengeS256()
}
if len(org.SSOConfig.OpenIdClientSecret) > 0 {
//baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&client_secret=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, org.SSOConfig.OpenIdClientSecret)
state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&redirect=%s&challenge=%s", org.Id, redirectUrl, org.SSOConfig.OpenIdClientSecret)))
baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=id_token&scope=openid email&redirect_uri=%s&state=%s&response_mode=form_post&nonce=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, state)
//baseSSOUrl += fmt.Sprintf("&client_secret=%s", org.SSOConfig.OpenIdClientSecret)
} else {
baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid email&redirect_uri=%s&state=%s&code_challenge_method=S256&code_challenge=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, codeChallenge)
}
return baseSSOUrl
}
/*
func HandleGenerateProvisionUrl(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting GET SUBORG request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
ctx := GetContext(request)
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Failed authentication in user provision: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Authentication required"}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Non-admin user %s attempted to provision user", user.Username)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Admin access required"}`))
return
}
// check if user is in a partner org
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed to get org for user %s: %s", user.Username, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to get organization"}`))
return
}
validationOrg := org
if len(org.CreatorOrg) > 0 {
parentOrg, err := GetOrg(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[WARNING] Failed to get parent org %s for user %s: %s", org.CreatorOrg, user.Username, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to get parent organization"}`))
return
}
validationOrg = parentOrg
}
isAdmin := false
for _, orgUser := range validationOrg.Users {
if orgUser.Id == user.Id && orgUser.Role == "admin" {
isAdmin = true
break
}
}
if !isAdmin {
log.Printf("[WARNING] User %s attempted to provision user without admin access in validation org %s", user.Username, validationOrg.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Admin access required in the relevant org"}`))
return
}
if !(validationOrg.LeadInfo.DistributionPartner || validationOrg.LeadInfo.IntegrationPartner || validationOrg.LeadInfo.ServicePartner || validationOrg.LeadInfo.TechPartner || validationOrg.LeadInfo.ChannelPartner) {
log.Printf("[WARNING] User %s attempted to provision user without partner access in validation org %s", user.Username, validationOrg.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Provisioning not allowed. We need a partner org for this."}`))
return
}
// check if auto provision is true (since this stupid variable is flipped)
if org.SSOConfig.AutoProvision {
log.Printf("[WARNING] User %s attempted to provision user with auto provision disabled", user.Username)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Provisioning not allowed. Auto provision is disabled."}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed to read body in user provision: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed to read request body"}`))
return
}
var provisionRequest struct {
Email string `json:"email"`
}
err = json.Unmarshal(body, &provisionRequest)
if err != nil {
log.Printf("[WARNING] Failed to parse provision request: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Invalid request format"}`))
return
}
provisionRequest.Email = strings.ToLower(strings.TrimSpace(provisionRequest.Email))
if !strings.Contains(provisionRequest.Email, "@") {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Invalid email address"}`))
return
}
if strings.Contains(provisionRequest.Email, "@shuffler.io") {
log.Printf("[WARNING] Attempted to provision @shuffler.io email: %s", provisionRequest.Email)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Cannot provision @shuffler.io email addresses"}`))
return
}
if len(org.SSOConfig.OpenIdClientId) == 0 || len(org.SSOConfig.OpenIdToken) == 0 {
log.Printf("[WARNING] SSO not configured for org %s", org.Id)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "SSO not configured for this organization"}`))
return
}
existingUser, err := GetUser(ctx, provisionRequest.Email)
if err != nil {
users, err := FindGeneratedUser(ctx, provisionRequest.Email)
if err != nil {
log.Printf("[ERROR] Failed to find generated user for email %s: %s", provisionRequest.Email, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed to find user"}`))
return
}
if len(users) > 1 {
log.Printf("[ERROR] (suspicious) Found multiple generated users for email %s", provisionRequest.Email)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed to find user"}`))
return
}
if len(users) == 1 {
existingUser = &users[0]
}
}
if existingUser.Id != "" {
if existingUser.ProvisionedByOrg == org.Id {
existingUser.InitSSOInfos()
ssoInfo, exists := existingUser.GetSSOInfo(org.Id)
log.Printf("[INFO] User ssoInfo: %+v", ssoInfo)
mode := ""
if exists && ssoInfo.Sub != "" {
mode = "login"
log.Printf("[INFO] User %s already has SSO configured, generating login URL", provisionRequest.Email)
} else {
log.Printf("[INFO] User %s was provisioned by org %s but SSO not completed, generating setup URL", provisionRequest.Email, org.Id)
}
ssoUrl, err := GetOpenIdUrl(request, *org, *existingUser, mode)
if err != nil {
log.Printf("[ERROR] Failed to generate SSO URL for existing user %s: %s", existingUser.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to generate SSO URL"}`))
return
}
log.Printf("[AUDIT] Admin %s generated SSO URL for user %s in org %s (mode: %s)", user.Username, provisionRequest.Email, org.Id, mode)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{
"success": true,
"sso_url": "%s",
"existing_user": true,
"sso_configured": %t
}`, ssoUrl, mode == "login")))
return
}
log.Printf("[WARNING] User %s already exists but wasn't provisioned by org %s", provisionRequest.Email, org.Id)
resp.WriteHeader(409)
resp.Write([]byte(`{"success": false, "reason": "User already exists"}`))
return
}
log.Printf("[AUDIT] User %s doesn't exist, creating new user", provisionRequest.Email)
newUser := new(User)
newUser.Username = provisionRequest.Email
newUser.GeneratedUsername = provisionRequest.Email
newUser.Password = uuid.NewV4().String()
newUser.Verified = true
newUser.Active = true
newUser.CreationTime = time.Now().Unix()
newUser.Orgs = []string{org.Id}
newUser.Role = "user"
newUser.Id = uuid.NewV4().String()
newUser.ProvisionedByOrg = org.Id
newUser.ActiveOrg = OrgMini{
Name: org.Name,
Id: org.Id,
Role: "user",
}
if project.Environment == "cloud" {
newUser.Regions = []string{"https://shuffler.io"}
if org.RegionUrl != "https://shuffler.io" {
newUser.Regions = append(newUser.Regions, org.RegionUrl)
}
}
err = SetUser(ctx, newUser, true)
if err != nil {
log.Printf("[ERROR] Failed to create user %s: %s", provisionRequest.Email, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to create user"}`))
return
}
// Add user to org
org.Users = append(org.Users, User{
Id: newUser.Id,
Username: newUser.Username,
Role: "user",
})
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[ERROR] Failed to update org %s after adding user: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to add user to organization"}`))
return
}
ssoUrl, err := GetOpenIdUrl(request, *org, *newUser, "")
if err != nil {
log.Printf("[ERROR] Failed to generate SSO URL for provisioned user %s: %s", newUser.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to generate SSO URL"}`))
return
}
log.Printf("[AUDIT] Admin %s (%s) provisioned user %s in org %s (%s)", user.Username, user.Id, provisionRequest.Email, org.Name, org.Id)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{
"success": true,
"sso_url": "%s",
"user_id": "%s",
}`, ssoUrl, newUser.Id)))
}
*/
func GetRequestIp(r *http.Request) string {
// Check the actual IP that is inbound
forwardedFor := r.Header.Get("X-Forwarded-For")
if forwardedFor != "" {
// The X-Forwarded-For header can contain a comma-separated list of IP addresses.
// The client's IP is usually the first one.
stringSplit := strings.Split(forwardedFor, ",")
if len(stringSplit) > 1 {
if debug {
log.Printf("[DEBUG] Found multiple IPs in X-Forwarded-For header: %s. Returning first.", forwardedFor)
}
return stringSplit[0]
} else {
return forwardedFor
}
}
// Check for the X-Real-IP header
realIP := r.Header.Get("X-Real-IP")
if realIP != "" {
if strings.Count(realIP, ":") == 1 {
return strings.Split(realIP, ":")[0]
}
return realIP
}
realIP = r.Header.Get("CF-Connecting-IP")
if realIP != "" {
if strings.Count(realIP, ":") == 1 {
return strings.Split(realIP, ":")[0]
}
return realIP
}
realIP = r.Header.Get("X-Appengine-User-Ip")
if realIP != "" {
if strings.Count(realIP, ":") == 1 {
return strings.Split(realIP, ":")[0]
}
return realIP
}
// Loop through and find headers with "IP" in them
for k, v := range r.Header {
if strings.Contains(strings.ToLower(k), "ip") {
log.Printf("[ERROR] Found useful unhandled IP header %s: %s", k, v)
}
}
// IPv6 / localhostm apping. Just returning raw.
if strings.Contains(r.RemoteAddr, "::") || strings.Contains(r.RemoteAddr, "127.0.0.1") || strings.Contains(r.RemoteAddr, "localhost") {
return r.RemoteAddr
}
// If neither header is present, fall back to using the RemoteAddr field.
// Check for IPv6 and split accordingly.
re := regexp.MustCompile(`\[[^\]]+\]`)
remoteAddr := re.ReplaceAllString(r.RemoteAddr, "")
if remoteAddr != "" {
return remoteAddr
}
remoteAddrSplit := strings.Split(r.RemoteAddr, ":")
return remoteAddrSplit[0]
}
func GetUserLocation(ctx context.Context, ip string) (UserGeoInfo, error) {
geoapifyKey := os.Getenv("GEOAPIFY_KEY")
if geoapifyKey == "" {
return UserGeoInfo{}, errors.New("GEOAPIFY_KEY is not set")
}
// Reject local or invalid IPs early
if strings.Contains(ip, "::1") || strings.Contains(ip, "127.0.0.1") || strings.Contains(ip, "localhost") || strings.Contains(ip, "[") || strings.Contains(ip, "]") {
//log.Printf("[DEBUG] Skipping Geoapify request : Invalid IP %s", ip)
return UserGeoInfo{}, errors.New("invalid ip")
}
cacheKey := fmt.Sprintf("geoinfo_%s", ip)
userGeoInfo, err := GetCache(ctx, cacheKey)
if err == nil {
var userLocationData UserGeoInfo
err = json.Unmarshal(userGeoInfo.([]byte), &userLocationData)
if err != nil {
log.Printf("[ERROR] Failed to parse user location data for IP %s: %s", ip, err)
return UserGeoInfo{}, err
}
return userLocationData, nil
}
url := fmt.Sprintf("https://api.geoapify.com/v1/ipinfo?apiKey=%s&ip=%s", geoapifyKey, ip)
resp, err := http.Get(url)
if err != nil {
log.Printf("[ERROR] Failed to get user location for IP %s: %s", ip, err)
return UserGeoInfo{}, err
}
defer resp.Body.Close()
// Handle non-200 responses
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body) // Read even if failed to get error message
log.Printf("[ERROR] Geoapify returned status %d for IP %s: %s", resp.StatusCode, ip, string(body))
return UserGeoInfo{}, errors.New("Geoapify returned status " + strconv.Itoa(resp.StatusCode))
}
var userLocationData UserGeoInfo
body, err := io.ReadAll(resp.Body)
if err != nil {
log.Printf("[ERROR] Failed to read user location data for IP %s: %s", ip, err)
return UserGeoInfo{}, err
}
err = json.Unmarshal(body, &userLocationData)
if err != nil {
log.Printf("[ERROR] Failed to parse user location data for IP %s: %s", ip, err)
return UserGeoInfo{}, err
}
err = SetCache(ctx, cacheKey, []byte(body), 60)
if err != nil {
log.Printf("[ERROR] Failed to cache user location data for IP %s: %s", ip, err)
}
return userLocationData, nil
}
func HandleLogin(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting LOGIN request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
err := ValidateRequestOverload(resp, request)
if err != nil {
log.Printf("[INFO] Request overload for IP %s in login", GetRequestIp(request))
resp.WriteHeader(429)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Too many requests"}`)))
return
}
// Gets a struct of Username, password
data, err := ParseLoginParameters(resp, request)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
ip := GetRequestIp(request)
log.Printf("[AUDIT] Handling login of username %s with IP: %s", data.Username, ip)
data.Username = strings.ToLower(strings.TrimSpace(data.Username))
err = CheckUsername(data.Username)
if err != nil {
log.Printf("[INFO] Username is too short or bad for %s: %s", data.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
ctx := GetContext(request)
users, err := FindUser(ctx, data.Username)
if err != nil && len(users) == 0 {
log.Printf("[WARNING] Failed getting user %s during login", data.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`))
return
}
userdata := User{}
if len(users) != 1 {
log.Printf("[WARNING] Username %s has multiple or no users (%d). Checking if it matches any.", data.Username, len(users))
for _, user := range users {
if user.Id == "" && user.Username == "" {
log.Printf(`[AUDIT] Username %s (%s) isn't valid (2). Amount of users checked: %d (1)`, user.Username, user.Id, len(users))
continue
}
if user.ActiveOrg.Id != "" {
err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(data.Password))
if err != nil {
log.Printf("[WARNING] Bad password: %s", err)
continue
}
userdata = user
break
}
}
} else {
userdata = users[0]
}
// Starting caching of the username
// This is to make it faster later :)
go GetAllWorkflowsByQuery(context.Background(), userdata, 250, "")
go GetPrioritizedApps(context.Background(), userdata)
/*
// FIXME: Reenable activation?
if project.Environment == "cloud" && !userdata.Active {
log.Printf("[DEBUG] %s is not active, but tried to login. Error: %v", data.Username, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This user is deactivated"}`))
return
}
*/
updateUser := false
if project.Environment == "cloud" {
if strings.HasSuffix(strings.ToLower(userdata.Username), "@shuffler.io") {
if !userdata.Active {
log.Printf("[INFO] User %s with @shuffler suffix is not active.", userdata.Username)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "error: You need to activate your account before logging in"}`)))
return
}
}
}
org, orgerr := GetOrg(ctx, userdata.ActiveOrg.Id)
if orgerr != nil && (len(org.Id) == 0 || len(org.Name) == 0) {
log.Printf("[ERROR] Failed getting active org '%s' during login for %s (%s). Remapping to another suborg if possible: %s", userdata.ActiveOrg.Id, userdata.Username, userdata.Id, orgerr)
for _, orgId := range userdata.Orgs {
innerorg, orgerr := GetOrg(ctx, orgId)
if orgerr != nil {
continue
}
if len(innerorg.Id) > 0 && len(innerorg.Name) > 0 {
userdata.ActiveOrg.Id = innerorg.Id
userdata.ActiveOrg.Name = innerorg.Name
org = innerorg
updateUser = true
break
}
}
if len(org.Id) == 0 {
log.Printf("[ERROR] Failed getting active org '%s' during login for %s (%s). Remapping to another suborg failed: %s", userdata.ActiveOrg.Id, userdata.Username, userdata.Id, orgerr)
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting org. If this persists, please contact support@shuffler.io"}`)))
return
}
}
changeActiveOrg := false
if orgerr == nil {
log.Printf("[DEBUG] Got org during signin: %s - checking SAML SSO", userdata.ActiveOrg.Id)
ssoRequired := org.SSOConfig.SSORequired
if ssoRequired {
orgFound := false
for _, orgString := range userdata.Orgs {
innerorg, err := GetOrg(ctx, orgString)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", orgString, err)
continue
}
if innerorg.SSOConfig.SSORequired {
continue
}
log.Printf("[INFO] Found Non-SSO org %s (%s) for user %s (%s)", innerorg.Name, innerorg.Id, userdata.Username, userdata.Id)
org = innerorg
userdata.ActiveOrg.Id = innerorg.Id
userdata.ActiveOrg.Name = innerorg.Name
orgFound = true
changeActiveOrg = true
break
}
baseSSOUrl := "" //org.SSOConfig.SSOEntrypoint
redirectKey := "SSO_REDIRECT"
if len(org.SSOConfig.OpenIdAuthorization) > 0 {
log.Printf("[INFO] OpenID login for %s", org.Id)
redirectKey = "SSO_REDIRECT"
baseSSOUrl = GetOpenIdUrl(request, *org)
}
if !orgFound && len(baseSSOUrl) > 0 {
log.Printf("[INFO] Inside SSO / OpenID check: %s", org.Id)
// has to contain http(s)
log.Printf("[DEBUG] Login: Should redirect user %s in org %s(%s) to SSO login at %s", userdata.Username, userdata.ActiveOrg.Name, userdata.ActiveOrg.Id, baseSSOUrl)
// Check if the user has other orgs that can be swapped to - if so SWAP
if !strings.HasPrefix(baseSSOUrl, "http") {
log.Printf("[ERROR] SSO URL for %s (%s) is invalid: %s", org.Name, org.Id, baseSSOUrl)
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false, "reason": "SSO URL is invalid"}`))
//return
} else {
// Check if the user has other orgs that can be swapped to - if so SWAP
log.Printf("[DEBUG] Change org: Should redirect user %s in org %s (%s) to SSO login at %s", userdata.Username, userdata.ActiveOrg.Name, userdata.ActiveOrg.Id, baseSSOUrl)
ssoResponse := SSOResponse{
Success: true,
Reason: redirectKey,
URL: baseSSOUrl,
}
b, err := json.Marshal(ssoResponse)
if err != nil {
log.Printf("[ERROR] Failed marshalling SSO response: %s", err)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(b)
return
}
}
}
}
if len(users) == 1 && len(data.Password) > 0 {
err = bcrypt.CompareHashAndPassword([]byte(userdata.Password), []byte(data.Password))
if err != nil {
userdata = User{}
log.Printf("[WARNING] Bad password: %s", err)
} else {
log.Printf("[DEBUG] Correct password with single user!")
}
}
if userdata.Id == "" && userdata.Username == "" {
log.Printf(`[AUDIT] Login for Username %s isn't valid with that password. Amount of users checked: %d (2)`, data.Username, len(users))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Username and/or password is incorrect"}`)))
return
}
if updateUser {
err = SetUser(ctx, &userdata, false)
if err != nil {
log.Printf("[WARNING] Failed updating user when auto-setting new org: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Something went wrong with the SSO redirect system"}`)))
return
}
}
// Preloading orgs into cache to speed up first requests a bit
for _, orgID := range userdata.Orgs {
go GetOrg(ctx, orgID)
}
if userdata.LoginType == "SSO" {
log.Printf(`[WARNING] Username %s (%s) has login type set to SSO (single sign-on).`, userdata.Username, userdata.Id)
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false, "reason": "This user can only log in with SSO"}`))
//return
}
if userdata.LoginType == "OpenID" {
log.Printf(`[WARNING] Username %s (%s) has login type set to OpenID (single sign-on).`, userdata.Username, userdata.Id)
}
if len(data.MFACode) == 0 {
log.Printf("[DEBUG] No MFA code found in login request for %s (%s). Checking %d orgs", userdata.Username, userdata.Id, len(userdata.Orgs))
for _, orgID := range userdata.Orgs {
org, err := GetOrg(ctx, orgID)
if err != nil {
log.Printf("[ERROR] Failed getting suborg %s during login for %s (%s): %s", orgID, userdata.Username, userdata.Id, err)
continue
}
if org.MFARequired {
if org.MFARequired && !userdata.MFA.Active {
log.Printf("MFA is required for org %s and user has not set up MFA.", orgID)
// Generate a unique code
MFACode := uuid.NewV4().String()
cacheKey := fmt.Sprintf("user_id_%s", MFACode)
err := SetCache(ctx, cacheKey, []byte(userdata.Id), 30)
if err != nil {
log.Printf("[ERROR] Failed setting cache for user %s: %s", userdata.Username, err)
continue
}
cacheKey = fmt.Sprintf("mfa_code_%s", MFACode)
err = SetCache(ctx, cacheKey, []byte(MFACode), 30)
if err != nil {
log.Printf("[ERROR] Failed setting cache for user %s: %s", userdata.Username, err)
continue
}
response := fmt.Sprintf(`{"success": true, "reason": "MFA_SETUP", "url": "%s"}`, MFACode)
resp.WriteHeader(200)
resp.Write([]byte(response))
return
}
log.Printf("[DEBUG] MFA is required for org %s. Redirecting.", orgID)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "MFA_REDIRECT"}`)))
return
}
}
}
if userdata.MFA.Active && len(data.MFACode) == 0 {
log.Printf(`[DEBUG] Username %s (%s) has MFA activated. Redirecting.`, userdata.Username, userdata.Id)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "MFA_REDIRECT"}`)))
return
}
if len(data.MFACode) > 0 && userdata.MFA.Active {
interval := time.Now().Unix() / 30
HOTP, err := getHOTPToken(userdata.MFA.ActiveCode, interval)
if err != nil {
log.Printf("[ERROR] Failed generating a HOTP token: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed generating token. Please try again."}`))
return
}
if HOTP != data.MFACode {
log.Printf("[DEBUG] Bad code sent for user %s (%s). Sent: %s, Want: %s", userdata.Username, userdata.Id, data.MFACode, HOTP)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Wrong 2-factor code (%s). Please try again with a 6-digit code. If this persists, please contact support."}`, data.MFACode)))
return
}
log.Printf("[DEBUG] MFA login for user %s (%s)!", userdata.Username, userdata.Id)
}
// This is a hack to get the real IP address
// https://stackoverflow.com/questions/27234861/golang-http-request-returns-127-0-0-1
userdata.LoginInfo = append(userdata.LoginInfo, LoginInfo{
IP: GetRequestIp(request),
Timestamp: time.Now().Unix(),
})
tutorialsFinished := []Tutorial{}
for _, tutorial := range userdata.PersonalInfo.Tutorials {
tutorialsFinished = append(tutorialsFinished, Tutorial{
Name: tutorial,
})
}
if len(org.Id) == 0 {
newOrg, err := GetOrg(ctx, userdata.ActiveOrg.Id)
if err == nil {
org = newOrg
}
}
if len(org.SecurityFramework.SIEM.Name) > 0 || len(org.SecurityFramework.Network.Name) > 0 || len(org.SecurityFramework.EDR.Name) > 0 || len(org.SecurityFramework.Cases.Name) > 0 || len(org.SecurityFramework.IAM.Name) > 0 || len(org.SecurityFramework.Assets.Name) > 0 || len(org.SecurityFramework.Intel.Name) > 0 || len(org.SecurityFramework.Communication.Name) > 0 {
tutorialsFinished = append(tutorialsFinished, Tutorial{
Name: "find_integrations",
})
}
for _, tutorial := range org.Tutorials {
tutorialsFinished = append(tutorialsFinished, tutorial)
}
//log.Printf("[INFO] Tutorials finished: %v", tutorialsFinished)
returnValue := HandleInfo{
Success: true,
Tutorials: tutorialsFinished,
}
loginData := `{"success": true}`
newData, err := json.Marshal(returnValue)
if err == nil {
loginData = string(newData)
}
// On cloud, we just generate a new org for them on the fly
// Onprem, the user shouldn't exist anymore, which means you would need to re-register. You should only get to this point if the user exists
if project.Environment != "cloud" {
// Check activeorg if they have access to it (the user)
found := false
foundOrg, err := GetOrg(ctx, userdata.ActiveOrg.Id)
if err == nil {
log.Printf("[DEBUG] Found org %s for user %s (%s).", userdata.ActiveOrg.Id, userdata.Username, userdata.Id)
for _, foundUser := range foundOrg.Users {
if foundUser.Id == userdata.Id {
found = true
break
}
}
if !found {
log.Printf("[DEBUG] Failed to find user %s (%s) in org %s", userdata.Username, userdata.Id, userdata.ActiveOrg.Id)
}
if !found && len(foundOrg.Users) == 0 {
// Forcefully add the user back in there (org)
err = fixOrgUsers(ctx, *foundOrg)
if err != nil {
log.Printf("[ERROR] Failed fixing org %s while re-adding a user: %s", foundOrg.Id, err)
}
}
} else {
log.Printf("[ERROR] Failed finding org %s during login: %s", userdata.ActiveOrg.Id, err)
}
// Check if we need to move them over (move the activeOrg to other user org)
if !found {
log.Printf("[DEBUG] Current active org (%s) for user %s (%s) not found. Checking other orgs. Found %d orgs.", userdata.ActiveOrg.Id, userdata.Username, userdata.Id, len(userdata.Orgs))
userdata.Role = "admin"
userdata.Roles = []string{"admin"}
for _, org := range userdata.Orgs {
// Verify if the user is in the org and if it points to an org but
// that does not exist we create re-create that org.
foundOrg, err := GetOrg(ctx, org)
if err != nil {
log.Printf("[WARNING] Failed finding org %s: %s. Trying to recreate the org", org, err)
rOrg := Org{
Name: "default",
Id: org,
Org: "default",
Users: []User{userdata},
Roles: userdata.Roles,
CloudSync: false,
}
err := SetOrg(ctx, rOrg, org)
found = true
if err != nil {
log.Printf("[ERROR] Failed to re-create the org")
found = false
}
log.Printf("[DEBUG] Re-created the org %s", org)
}
for _, foundUser := range foundOrg.Users {
if foundUser.Id == userdata.Id {
found = true
break
}
}
if found {
break
}
}
}
// User has no orgs after all checks, create a default
if !found {
// Check all the workflows has orgs and user
workflows, err := GetAllWorkflows(ctx)
log.Printf("[DEBUG] Checking all the worflows and finding user a org.")
userdata.Role = "admin"
userdata.Roles = []string{"admin"}
for _, workflow := range workflows {
for _, vOrg := range workflow.Org {
wOrg, err := GetOrg(ctx, vOrg.Id)
if err != nil {
log.Printf("[WARNING] Faild getting a org %s for a workflow %s", vOrg.Id, workflow.ID)
log.Printf("[DEBUG] Recreating the org %s", vOrg.Id)
WorkflowOrg := Org{
Name: vOrg.Name,
Id: vOrg.Id,
Org: vOrg.Name,
Users: []User{},
Roles: []string{vOrg.Role},
CloudSync: false,
}
err := SetOrg(ctx, WorkflowOrg, vOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed setting a org")
}
}
err = fixOrgUsers(ctx, *wOrg)
if err != nil {
log.Printf("[ERROR] %s", err)
}
}
}
log.Printf("[WARNING] User %s (%s) has no orgs. ID: %s, Name: %s. Creating a default one.", userdata.Username, userdata.Id, userdata.ActiveOrg.Id, userdata.ActiveOrg.Name)
orgSetupName := "default"
orgId := uuid.NewV4().String()
newOrg := Org{
Name: orgSetupName,
Id: orgId,
Org: orgSetupName,
Users: []User{userdata},
Roles: userdata.Roles,
CloudSync: false,
}
err = SetOrg(ctx, newOrg, newOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed setting default org for the user: %s", userdata.Username)
} else {
log.Printf("[DEBUG] Successfully created the default org!")
defaultEnv := os.Getenv("ORG_ID")
if len(defaultEnv) == 0 {
defaultEnv = "Shuffle"
log.Printf("[DEBUG] Setting default environment for org to %s", defaultEnv)
}
item := Environment{
Name: defaultEnv,
Type: "onperm",
OrgId: orgId,
Default: true,
Id: uuid.NewV4().String(),
}
err := SetEnvironment(ctx, &item)
if err != nil {
log.Printf("[ERROR] Failed setting up new environment for new org: %s", err)
}
userdata.Orgs = append(userdata.Orgs, newOrg.Id)
}
userdata.ActiveOrg.Id = userdata.Orgs[0]
}
}
regionUrl := ""
if project.Environment == "cloud" {
if len(userdata.ActiveOrg.RegionUrl) > 0 {
regionUrl = userdata.ActiveOrg.RegionUrl
} else {
org, err := GetOrg(ctx, userdata.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s during login for %s (%s): %s", userdata.ActiveOrg.Id, userdata.Username, userdata.Id, err)
} else {
if strings.Contains(strings.ToLower(org.RegionUrl), "http") {
regionUrl = strings.ToLower(org.RegionUrl)
}
}
}
}
// Had to set this due to session hashing rollback
if len(userdata.Session) != 0 && len(userdata.Session) == 36 && !changeActiveOrg {
log.Printf("[INFO] User session exists - resetting session")
expiration := time.Now().Add(8 * time.Hour)
newCookie := ConstructSessionCookie(userdata.Session, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
//log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", userdata.Session)
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "session_token",
Value: userdata.Session,
Expiration: expiration.Unix(),
})
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "__session",
Value: userdata.Session,
Expiration: expiration.Unix(),
})
// Singul handler
if project.Environment == "cloud" {
newCookie.Name = "__session"
newCookie.Domain = ".singul.io"
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
newCookie.Domain = ".shutdown.no"
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
newCookie.Domain = ".shuffler.io"
http.SetCookie(resp, newCookie)
}
loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}], "region_url": "%s"}`, userdata.Session, expiration.Unix(), regionUrl)
newData, err := json.Marshal(returnValue)
if err == nil {
loginData = string(newData)
}
err = SetSession(ctx, userdata, userdata.Session)
if err != nil {
log.Printf("[WARNING] Error adding session to database: %s", err)
} else {
//log.Printf("[DEBUG] Updated session in backend")
}
err = SetUser(ctx, &userdata, false)
if err != nil {
log.Printf("[ERROR] Failed updating user when setting session (2): %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(loginData))
return
} else {
log.Printf("[INFO] User session for %s (%s) is empty - create one!", userdata.Username, userdata.Id)
sessionToken := uuid.NewV4().String()
expiration := time.Now().Add(8 * time.Hour)
newCookie := ConstructSessionCookie(sessionToken, expiration)
// Does it not set both?
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
// ADD TO DATABASE
err = SetSession(ctx, userdata, sessionToken)
if err != nil {
log.Printf("[DEBUG] Error adding session to database: %s", err)
}
userdata.Session = sessionToken
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "session_token",
Value: sessionToken,
Expiration: expiration.Unix(),
})
returnValue.Cookies = append(returnValue.Cookies, SessionCookie{
Key: "__session",
Value: sessionToken,
Expiration: expiration.Unix(),
})
// Singul handler
if project.Environment == "cloud" {
newCookie.Name = "__session"
newCookie.Domain = ".singul.io"
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
newCookie.Domain = ".shutdown.no"
http.SetCookie(resp, newCookie)
}
err = SetUser(ctx, &userdata, true)
if err != nil {
log.Printf("[ERROR] Failed updating user when setting session: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}], "region_url": "%s"}`, sessionToken, expiration.Unix(), regionUrl)
newData, err := json.Marshal(returnValue)
if err == nil {
loginData = string(newData)
}
}
log.Printf("[AUDIT] Login successful for user %s (%s) with IP: %s, session: %s", userdata.Username, userdata.Id, ip, userdata.Session)
resp.WriteHeader(200)
resp.Write([]byte(loginData))
}
// FIXME: Do NOT use this yet (May 24th, 2024). It is not ready for production due to being a potential cross-tenant attack vector.
func HandleSAML(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte("SAML SSO is deprecated. Please use OpenID Connect instead. Contact support@shuffler.io if you need help migrating, or are having trouble logging in to your account."))
return
}
func ParseLoginParameters(resp http.ResponseWriter, request *http.Request) (loginStruct, error) {
if request.Body == nil {
return loginStruct{}, errors.New("Failed to parse login params, body is empty")
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
return loginStruct{}, err
}
var t loginStruct
err = json.Unmarshal(body, &t)
if err != nil {
return loginStruct{}, err
}
return t, nil
}
func CheckUsername(Username string) error {
// Stupid first check of email loool
//if !strings.Contains(Username, "@") || !strings.Contains(Username, ".") {
// return errors.New("Invalid Username")
//}
if len(Username) < 3 {
return errors.New("Minimum Username length is 3")
}
return nil
}
// Handles workflow executions across systems (open source, worker, cloud)
// getWorkflow
// GetWorkflow
// executeWorkflow
// This should happen locally.. Meaning, polling may be stupid.
// Let's do it anyway, since it seems like the best way to scale
// without remoting problems and the like.
func updateExecutionParent(ctx context.Context, executionParent, returnValue, parentAuth, parentNode, subflowExecutionId string) error {
// Was an error here. Now defined to run with http://shuffle-backend:5001 by default
backendUrl := os.Getenv("BASE_URL")
if project.Environment == "cloud" {
backendUrl = "https://shuffler.io"
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
}
// FIXME: This MAY fail at scale due to not being able to get the right worker
// Maybe we need to pass the worker's real id, and not its VIP?
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
backendUrl = "http://shuffle-workers:33333"
hostenv := os.Getenv("WORKER_HOSTNAME")
if len(hostenv) > 0 {
backendUrl = fmt.Sprintf("http://%s:33333", hostenv)
}
// From worker:
//parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport)
log.Printf("[DEBUG][%s] Sending request for shuffle-subflow result to %s. Should this be a specific worker? Specific worker is better if cache is NOT memcached", subflowExecutionId, backendUrl)
}
// Waiting due to speed problems in certain circumstances
time.Sleep(350 * time.Millisecond)
// Callback to itself
if len(backendUrl) == 0 {
backendUrl = "http://localhost:5001"
}
resultUrl := fmt.Sprintf("%s/api/v1/streams/results", backendUrl)
topClient := GetExternalClient(backendUrl)
newExecution := WorkflowExecution{}
requestData := ActionResult{
Authorization: parentAuth,
ExecutionId: executionParent,
}
data, err := json.Marshal(requestData)
if err != nil {
log.Printf("[WARNING] Failed parent init marshal: %s", err)
return err
}
req, err := http.NewRequest(
"POST",
resultUrl,
bytes.NewBuffer([]byte(data)),
)
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Failed making parent request: %s. Is URL valid: %s", err, backendUrl)
return err
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading parent body: %s", err)
return err
}
if newresp.StatusCode != 200 {
log.Printf("[ERROR] Bad statuscode setting subresult (1) with URL %s: %d, %s. Input data: %s", resultUrl, newresp.StatusCode, string(body), string(data))
return errors.New(fmt.Sprintf("Bad statuscode: %d", newresp.StatusCode))
}
err = json.Unmarshal(body, &newExecution)
if err != nil {
log.Printf("[ERROR] Failed newexecutuion parent unmarshal: %s", err)
return err
}
foundResult := ActionResult{}
for _, result := range newExecution.Results {
if result.Action.ID == parentNode {
foundResult = result
break
}
}
isLooping := false
selectedTrigger := Trigger{}
// Validating parent node
checkResult := false
// Subflows and the like may not be in here anymore. Maybe they are in actions
for _, trigger := range newExecution.Workflow.Triggers {
if trigger.ID != parentNode {
continue
}
selectedTrigger = trigger
for _, param := range trigger.Parameters {
if param.Name == "argument" && isLoop(param.Value) {
// Check if the .# exists, without .#0 or .#1 for digits
//re := regexp.MustCompile(`\.\#(\d+)`)
//if re.MatchString(param.Value) {
// log.Printf("\n\n\n[DEBUG][%s] Found a loop in the subflow. Not mapping subflow result back to parent workflow. Trigger: %#v\n\n\n", subflowExecutionId, selectedTrigger.ID)
//}
isLooping = true
}
// Check for if wait for results is set
if param.Name == "check_result" {
if param.Value == "true" {
checkResult = true
} else {
checkResult = false
}
}
}
break
}
// Because we changed out how we handle mid-flow triggers
if len(selectedTrigger.ID) == 0 {
for _, action := range newExecution.Workflow.Actions {
if action.ID != parentNode {
continue
}
selectedTrigger = Trigger{
ID: action.ID,
Label: action.Label,
}
foundResult.Action = action
for _, param := range action.Parameters {
if param.Name == "argument" && isLoop(param.Value) {
isLooping = true
}
// Check for if wait for results is set
if param.Name == "check_result" {
if param.Value == "true" {
checkResult = true
} else {
checkResult = false
}
}
}
break
}
}
// Checks if the variable is set properly
if !checkResult {
//log.Printf("[DEBUG][%s] No check_result param found for subflow. Not mapping subflow result back to parent workflow. Trigger: %#v", subflowExecutionId, selectedTrigger.ID)
return nil
}
// IF the workflow is looping, the result is added in the backend to not
// cause consistency issues. This means the result will be sent back, and instead
// Added to the workflow result by the backend itself.
// When all the "WAITING" executions are done, the backend will set the execution itself
// back to executing, allowing the parent to continue
sendRequest := false
resultData := []byte{}
if isLooping {
//log.Printf("[DEBUG][%s] SUBFLOW LOOPING - SHOULD ADD TO A LIST!", subflowExecutionId)
// Saved for each subflow ID -> parentNode
subflowResultCacheId := fmt.Sprintf("%s_%s_subflowresult", subflowExecutionId, parentNode)
if len(returnValue) > 0 {
err = SetCache(ctx, subflowResultCacheId, []byte(returnValue), 61)
if err != nil {
log.Printf("[ERROR] Failed setting subflow loop cache result for action in parsed exec results %s: %s", subflowResultCacheId, err)
return err
}
}
// Every time we get here, we need to both SET the value in cache AND look for other values in cache to make sure the list is good.
parentNodeFound := false
var parentSubflowResult []SubflowData
for _, result := range newExecution.Results {
if result.Action.ID != parentNode {
continue
}
//log.Printf("[DEBUG] FOUND RES: %s", foundResult.Result)
parentNodeFound = true
err = json.Unmarshal([]byte(foundResult.Result), &parentSubflowResult)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal result to parentsubflow res: %s", err)
continue
}
break
}
// If found, loop through and make sure to check the result for ALL of them. If they're not in there, add them as values.
if parentNodeFound {
log.Printf("[DEBUG] Found result for subflow (parentNodeFound). Got %d parentSubflowResults", len(parentSubflowResult))
ranUpdate := false
newResults := []SubflowData{}
finishedSubflows := 0
for _, res := range parentSubflowResult {
// If value length = 0 for any, then check cache and add the result
//log.Printf("[DEBUG] EXEC: %s", res)
if res.ExecutionId == subflowExecutionId {
//foundResult.Result
res.Result = string(returnValue)
res.ResultSet = true
ranUpdate = true
//log.Printf("[DEBUG] Set the result for the node! Run update with %s", res)
finishedSubflows += 1
} else {
res.ResultSet = true
// Overutilization of cache :>
if !res.ResultSet || len(res.Result) == 0 {
subflowResultCacheId = fmt.Sprintf("%s_%s_subflowresult", res.ExecutionId, parentNode)
cache, err := GetCache(ctx, subflowResultCacheId)
if err == nil {
cacheData := []byte(cache.([]uint8))
res.Result = string(cacheData)
res.ResultSet = true
ranUpdate = true
finishedSubflows += 1
} else {
//log.Printf("[DEBUG] No cache data set for subflow cache %s", subflowResultCacheId)
}
} else {
finishedSubflows += 1
}
}
newResults = append(newResults, res)
}
log.Printf("[INFO][%s] TOTAL FINISHED SUBFLOWS: %d/%d", subflowExecutionId, len(parentSubflowResult), len(newResults))
// Can it be if this and also status = "WAITING"?
if len(parentSubflowResult) == finishedSubflows && foundResult.Status != "SUCCESS" && foundResult.Status != "FAILURE" {
log.Printf("[INFO][%s] ALL THE SUBFLOW GOT THE RESULT BACK SO UPATING THE STATUS TO SUCCESS", subflowExecutionId)
foundResult.Status = "SUCCESS"
if foundResult.CompletedAt == 0 {
foundResult.CompletedAt = time.Now().Unix() * 1000
}
ranUpdate = true
sendRequest = true
}
if ranUpdate {
// FIXME: Look into whether this sendRequest can be removed if we want reduce the amount of request
sendRequest = true
baseResultData, err := json.Marshal(newResults)
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling subflow loop request data (1): %s", subflowExecutionId, err)
return err
}
foundResult.Result = string(baseResultData)
foundResult.ExecutionId = executionParent
foundResult.Authorization = parentAuth
resultData, err = json.Marshal(foundResult)
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling FULL subflow loop request data (2): %s", subflowExecutionId, err)
return err
}
}
} else {
//log.Printf("[ERROR][%s] Did NOT enter parentNodeFound in subflow loop. This means we can't update the parent", subflowExecutionId)
}
// Check if the item alreayd exists or not in results
//return nil
} else {
//log.Printf("\n\n[DEBUG] ITS NOT LOOP for parent node '%s'. Found data: %s\n\n", parentNode, returnValue)
if len(selectedTrigger.ID) > 0 {
foundResult.Action.ID = selectedTrigger.ID
}
// 1. Get result of parentnode's subflow (foundResult.Result)
// 2. Try to marshal parent into a loop.
// 3. If possible, loop through and find the one matching SubflowData.ExecutionId with "executionParent"
// 4. If it's matching, update ONLY that one.
// Why are we unmarshling it as array if not loop? It could cause inconsistency
var subflowDataLoop []SubflowData
err = json.Unmarshal([]byte(foundResult.Result), &subflowDataLoop)
if err == nil && len(subflowDataLoop) > 1 {
for subflowIndex, subflowData := range subflowDataLoop {
if subflowData.ExecutionId == executionParent {
log.Printf("[DEBUG][%s] Updating execution Id %s with subflow info", subflowExecutionId, subflowData.ExecutionId)
subflowDataLoop[subflowIndex].Result = returnValue
}
}
//foundResult.ExecutionId = executionParent
//foundResult.Authorization = parentAuth
resultData, err = json.Marshal(subflowDataLoop)
if err != nil {
log.Printf("[WARNING] Failed updating resultData (4): %s", err)
return err
}
sendRequest = true
} else {
// In here maning no-loop?
/*
actionValue := SubflowData{
Success: true,
ExecutionId: executionParent,
Authorization: parentAuth,
Result: returnValue,
}
*/
actionValue := SubflowData{
Success: true,
ExecutionId: "",
Authorization: "",
Result: returnValue,
}
newCacheKey := fmt.Sprintf("%s_%s_sinkholed_result", executionParent, parentNode)
cacheData, err := GetCache(ctx, newCacheKey)
if err != nil {
log.Printf("[ERROR][%s] Failed Getting sinkholed cache for subflow action result (4): %s", subflowExecutionId, err)
} else {
mappedData, ok := cacheData.([]byte)
if ok {
// Unmarshal it into actionValue
err = json.Unmarshal(mappedData, &actionValue)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling cache for subflow action result %s (4): %s", subflowExecutionId, err)
if debug {
log.Printf("\n\nSinkholed result DATA: %s\n\n", string(mappedData))
}
}
} else {
log.Printf("[ERROR] Failed type assertion for subflow action result %s (4): %s", subflowExecutionId, err)
}
}
// Keep the original info
// Where is it kept? :thinking:
actionValue.Result = returnValue
parsedActionValue, err := json.Marshal(actionValue)
if err != nil {
log.Printf("[ERROR] Failed updating resultData (1): %s", err)
return err
}
// This is probably bad for loops
timeNow := time.Now().Unix()
if len(foundResult.Action.ID) == 0 {
log.Printf("\n\n[INFO] Couldn't find the result? Data: %s\n\n", string(resultData))
parsedAction := Action{
Label: selectedTrigger.Label,
ID: parentNode,
Name: "run_subflow",
AppName: "shuffle-subflow",
AppVersion: "1.1.0",
Environment: selectedTrigger.Environment,
ExecutionDelay: selectedTrigger.ExecutionDelay,
}
newResult := ActionResult{
Action: parsedAction,
ExecutionId: executionParent,
Authorization: parentAuth,
Result: string(parsedActionValue),
StartedAt: timeNow,
CompletedAt: timeNow,
Status: "SUCCESS",
}
resultData, err = json.Marshal(newResult)
if err != nil {
log.Printf("[ERROR] Failed updating resultData (2): %s", err)
return err
}
sendRequest = true
} else {
if debug {
log.Printf("[DEBUG][%s] Found subflow result. Sending result with input data", foundResult.ExecutionId)
}
foundResult.StartedAt = timeNow
foundResult.CompletedAt = timeNow
foundResult.Authorization = parentAuth
foundResult.ExecutionId = executionParent
foundResult.Result = string(parsedActionValue)
if foundResult.Status == "" {
foundResult.Status = "SUCCESS"
}
resultData, err = json.Marshal(foundResult)
if err != nil {
log.Printf("[ERROR][%s] Failed updating resultData (3): %s", subflowExecutionId, err)
return err
}
sendRequest = true
}
}
}
// This is to ensure cache is in time. Timing issues between parent & child nodes are awful :)
if isLooping {
cacheId := fmt.Sprintf("%s_%s_result", foundResult.ExecutionId, foundResult.Action.ID)
err = SetCache(ctx, cacheId, resultData, 35)
if err != nil {
log.Printf("[WARNING][%s] Couldn't set cache for subflow action result %s (4): %s", subflowExecutionId, cacheId, err)
}
log.Printf("[DEBUG][%s] Set cache for subflow action result loop %s (4) with 250 ms delay before request", subflowExecutionId, cacheId)
//time.Sleep(250 * time.Millisecond)
}
if sendRequest && len(resultData) > 0 {
//log.Printf("[INFO][%s] Should send subflow request to backendURL %s. Data: %s!", executionParent, backendUrl, string(resultData))
// if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
// backendUrl = os.Getenv("BASE_URL")
// if project.Environment == "cloud" {
// backendUrl = "https://shuffler.io"
//
// if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
// backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
// }
//
// if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
// backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
// }
//
// }
// }
streamUrl := fmt.Sprintf("%s/api/v1/streams", backendUrl)
req, err := http.NewRequest(
"POST",
streamUrl,
bytes.NewBuffer([]byte(resultData)),
)
if err != nil {
log.Printf("[ERROR] Error building subflow (%s) request: %s", subflowExecutionId, err)
return err
}
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Error running subflow (%s) request: %s", subflowExecutionId, err)
return err
}
defer newresp.Body.Close()
if newresp.StatusCode != 200 {
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[INFO][%s] Failed reading body after subflow request: %s", subflowExecutionId, err)
return err
} else {
log.Printf("[ERROR][%s] Failed forwarding subflow request of length %d\n: %s", subflowExecutionId, len(resultData), string(body))
}
}
} else {
log.Printf("[INFO][%s] NOT sending request to parent %s because data len is %d and sendRequest is %t", subflowExecutionId, executionParent, len(resultData), sendRequest)
}
return nil
}
func ResendActionResult(actionData []byte, retries int64) {
if project.Environment == "cloud" && retries == 0 {
retries = 4
//return
//var res ActionResult
//err := json.Unmarshal(actionData, &res)
//if err == nil {
// log.Printf("[WARNING] Cloud - skipping rerun with %d retries for %s (%s)", retries, res.Action.Label, res.Action.ID)
//}
//return
}
if retries >= 5 {
return
}
backendUrl := os.Getenv("BASE_URL")
if project.Environment == "cloud" {
backendUrl = "https://shuffler.io"
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
backendUrl = "http://shuffle-workers:33333"
// Should connect to self, not shuffle-workers
hostenv := os.Getenv("WORKER_HOSTNAME")
if len(hostenv) > 0 {
backendUrl = fmt.Sprintf("http://%s:33333", hostenv)
}
//parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport)
// From worker:
//parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport)
log.Printf("\n\n[DEBUG] REsending request to rerun action result to %s\n\n", backendUrl)
// Here to prevent infinite loops
var res ActionResult
err := json.Unmarshal(actionData, &res)
if err == nil {
ctx := context.Background()
parsedValue, err := GetBackendexecution(ctx, res.ExecutionId, res.Authorization)
if err != nil {
log.Printf("[WARNING] Failed getting execution from backend to verify (3): %s", err)
} else {
log.Printf("[INFO][%s] Found execution result (3) %s for subflow %s in backend with %d results and result %s", res.ExecutionId, parsedValue.Status, res.ExecutionId, len(parsedValue.Results), parsedValue.Result)
if parsedValue.Status != "EXECUTING" {
return
}
}
}
}
if len(backendUrl) == 0 {
backendUrl = "http://localhost:5001"
}
//log.Printf("[INFO] Resending action result to backend %s", backendUrl)
log.Printf("[DEBUG][] Action: Resend, Label: '', Action: '', Status: '', Run status: '', Extra=url:%s", backendUrl)
streamUrl := fmt.Sprintf("%s/api/v1/streams?rerun=true&retries=%d", backendUrl, retries+1)
req, err := http.NewRequest(
"POST",
streamUrl,
bytes.NewBuffer(actionData),
)
if err != nil {
log.Printf("[ERROR] Error building resend action request - retries: %d, err: %s", retries, err)
if project.Environment != "cloud" && retries < 5 {
if strings.Contains(fmt.Sprintf("%s", err), "cannot assign requested address") {
time.Sleep(5 * time.Second)
retries = retries + 1
ResendActionResult(actionData, retries)
}
}
return
}
//Timeout: 3 * time.Second,
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
newresp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Error running resend action request - retries: %d, err: %s", retries, err)
if !strings.Contains(fmt.Sprintf("%s", err), "context deadline") && !strings.Contains(fmt.Sprintf("%s", err), "Client.Timeout exceeded") {
// How to self repair? Quit and restart the worker?
// This means worker is buggy when talking to itself
if project.Environment != "cloud" && retries < 5 {
if strings.Contains(fmt.Sprintf("%s", err), "cannot assign requested address") {
time.Sleep(5 * time.Second)
retries = retries + 1
ResendActionResult(actionData, retries)
}
} else if project.Environment != "cloud" && retries >= 5 {
//panic("No more sockets available. Restarting worker to self-repair.")
log.Printf("[WARNING] Should we quit out on worker and start a new? How can we remove socket boundry?")
}
}
return
}
defer newresp.Body.Close()
//body, err := ioutil.ReadAll(newresp.Body)
//if err != nil {
// log.Printf("[WARNING] Error getting body from rerun: %s", err)
// return
//}
//log.Printf("[DEBUG] Status %d and Body from rerun: %s", newresp.StatusCode, string(body))
}
// Function for translating action results into whatever.
// Came about because of general issues with Oauth2
func FixActionResultOutput(actionResult ActionResult) ActionResult {
if strings.Contains(actionResult.Result, "TypeError") && strings.Contains(actionResult.Result, "missing 1 required positional argument: 'access_token'") {
//log.Printf("\n\nTypeError in actionresult!")
actionResult.Result = `{"success": false, "reason": "This App requires authentication with Oauth2. Make sure to authenticate it first.", "extra": "If the app is authenticated, are you sure the Token & Refresh URL in the App is correct? Authentication refresh may have failed."}`
}
// Check length of result timestamp
if len(strconv.FormatInt(actionResult.StartedAt, 10)) == 10 {
actionResult.StartedAt = actionResult.StartedAt * 1000
}
if len(strconv.FormatInt(actionResult.CompletedAt, 10)) == 10 {
actionResult.CompletedAt = actionResult.CompletedAt * 1000
}
if len(strconv.FormatInt(actionResult.StartedAt, 10)) == 19 {
actionResult.StartedAt = actionResult.StartedAt / 1000000
}
if len(strconv.FormatInt(actionResult.CompletedAt, 10)) == 19 {
actionResult.CompletedAt = actionResult.CompletedAt / 1000000
}
//log.Printf("[DEBUG] Fixed LEN: %d, %d", len(strconv.FormatInt(actionResult.StartedAt, 10)), len(strconv.FormatInt(actionResult.CompletedAt, 10)))
return actionResult
}
func runTranslation(ctx context.Context, standard string, inputBody string) {
// Send HTTP request to localhost:3001
httpClient := &http.Client{}
url := fmt.Sprintf("http://localhost:5003/api/v1/translate_to/%s", standard)
req, err := http.NewRequest(
"POST",
url,
bytes.NewBuffer([]byte(inputBody)),
)
if err != nil {
log.Printf("[WARNING] Error building translation request to %s: %s", standard, err)
return
}
newresp, err := httpClient.Do(req)
if err != nil {
log.Printf("[WARNING] Error running translation to %s: %s", standard, err)
return
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[WARNING] Error getting body from translation for %s: %s", standard, err)
return
}
log.Printf("[DEBUG] Status: %d", newresp.StatusCode)
log.Printf("\n\n\nOUTPUT: %s\n\n\n", string(body))
}
func RunExecutionTranslation(ctx context.Context, actionResult ActionResult) {
//log.Printf("\n\n[DEBUG] Running execution translation for app '%s' with action '%s' towards standardized data\n\n", actionResult.Action.AppName, actionResult.Action.Name)
return
// Try to unmarshal the data to see if it has a status and if its less than 300
var parsedValue map[string]interface{}
err := json.Unmarshal([]byte(actionResult.Result), &parsedValue)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling action result for translation: %s", err)
return
}
// For now only handling proper returns with standard HTTP messaging
if status, ok := parsedValue["status"]; ok {
if status == nil {
log.Printf("[DEBUG] Found BAD status in action result: %s", status)
return
}
parsedStatus := status.(float64)
if parsedStatus >= 300 {
log.Printf("[DEBUG] Found status in action result: %f", parsedStatus)
return
}
} else {
log.Printf("[DEBUG] Did NOT find status in action result: %s", actionResult.Result)
return
}
log.Printf("\n\n[DEBUG] Running execution translation for app '%s' with action '%s' towards standardized data\n\n", actionResult.Action.AppName, actionResult.Action.Name)
parsedBody := ""
if body, ok := parsedValue["body"]; ok {
if body == nil {
return
}
// Check if body is a dictionary
if reflect.TypeOf(body).Kind() == reflect.Map {
bodyDict := body.(map[string]interface{})
bodyDictBytes, err := json.Marshal(bodyDict)
if err != nil {
log.Printf("[WARNING] Failed marshalling body dict: %s", err)
return
}
parsedBody = string(bodyDictBytes)
// Look for a list of items in here?
// How does it know to look for a list?
runTranslation(ctx, "email", parsedBody)
} else if reflect.TypeOf(body).Kind() == reflect.Slice {
// Check if body is a list and marshal it
bodyList := body.([]interface{})
bodyListBytes, err := json.Marshal(bodyList)
if err != nil {
log.Printf("[WARNING] Failed marshalling body list: %s", err)
return
}
// Should loop the items?
//parsedBody = string(bodyListBytes)
log.Printf("[WARNING] Found body list in action result of length: %d. Warning: No handler of lists yet", len(bodyListBytes))
}
}
//log.Printf("\n\n[DEBUG] Found body in action result of length: %d", len(parsedBody))
}
func sendAgentActionSelfRequest(status string, workflowExecution WorkflowExecution, actionResult ActionResult) error {
if project.Environment == "worker" {
return nil
}
ctx := context.Background()
// Check if the request has been sent already (just in case)
cacheKey := fmt.Sprintf("agent_request_%s_%s_%s", workflowExecution.ExecutionId, actionResult.Action.ID, status)
// cacheKey := fmt.Sprintf("agent_request_%s_%s", workflowExecution.ExecutionId, actionResult.Action.ID)
_, err := GetCache(ctx, cacheKey)
if err == nil {
//if debug {
// log.Printf("[DEBUG][%s] Agent self-request for Agent Result '%s' with status '%s' has already been sent. Skipping.", workflowExecution.ExecutionId, actionResult.Action.ID, status)
//}
return nil
} else {
var cacheTTL int32 = 1 // 1 minute for non-terminal statuses
if status == "SUCCESS" || status == "FINISHED" || status == "FAILURE" || status == "ABORTED" {
cacheTTL = 1440 // 24 hours — execution outcome is permanent
}
SetCache(ctx, cacheKey, []byte("1"), cacheTTL)
// SetCache(ctx, cacheKey, []byte(status), cacheTTL)
}
if status == "SUCCESS" || status == "FINISHED" || status == "FAILURE" || status == "ABORTED" {
agentOut := AgentOutput{}
json.Unmarshal([]byte(actionResult.Result), &agentOut)
duration := int64(0)
if agentOut.StartedAt > 0 && agentOut.CompletedAt > 0 {
duration = agentOut.CompletedAt - agentOut.StartedAt
} else if agentOut.StartedAt > 0 {
duration = time.Now().Unix() - agentOut.StartedAt
}
log.Printf("[INFO] AI_AGENT_FINISH: execution_id=%s org=%s status=%s duration=%ds decisions=%d llm_calls=%d tokens_used=%d", workflowExecution.ExecutionId, workflowExecution.Workflow.OrgId, status, duration, len(agentOut.Decisions), agentOut.LLMCallCount, agentOut.TotalTokens)
}
//log.Printf("[INFO][%s] Sending self-request for Agent Result '%s'. Status: %s", workflowExecution.ExecutionId, actionResult.Action.ID, status)
fixedActionResult := AgentOutput{}
err = json.Unmarshal([]byte(actionResult.Result), &fixedActionResult)
if err == nil && fixedActionResult.Status != "" {
if fixedActionResult.Status == "RUNNING" {
if status == "FINISHED" {
fixedActionResult.Status = "FINISHED"
} else if status == "ABORTED" || status == "FAILURE" {
fixedActionResult.Status = "FAILURE"
}
}
if status == "FINISHED" {
fixedActionResult.CompletedAt = time.Now().Unix()
} else if status == "ABORTED" || status == "FAILURE" {
fixedActionResult.CompletedAt = time.Now().Unix()
fixedActionResult.Error = "Agent decision was aborted or failed. Check the last decision for more information."
}
marshalledResult, err := json.Marshal(fixedActionResult)
if err == nil {
actionResult.Result = string(marshalledResult)
}
}
actionResult.ExecutionId = workflowExecution.ExecutionId
actionResult.Authorization = workflowExecution.Authorization
actionResult.Status = status
timenow := time.Now().UnixMicro()
if actionResult.StartedAt == 0 {
actionResult.StartedAt = timenow
}
actionResult.CompletedAt = timenow
baseUrl := fmt.Sprintf("https://shuffler.io")
if len(os.Getenv("BASE_URL")) > 0 {
baseUrl = os.Getenv("BASE_URL")
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
marshalledResult, err := json.Marshal(actionResult)
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling failure request for agent: %s", workflowExecution.ExecutionId, err)
return err
}
actionResultCacheId := fmt.Sprintf("%s_%s_result", actionResult.ExecutionId, actionResult.Action.ID)
go SetCache(context.Background(), actionResultCacheId, marshalledResult, 35)
fullUrl := fmt.Sprintf("%s/api/v1/streams", baseUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
bytes.NewBuffer(marshalledResult),
)
if err != nil {
log.Printf("[ERROR][%s] Error building agent '%s' request: %s", workflowExecution.ExecutionId, status, err)
return err
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Error running agent '%s' request (%s): %s", workflowExecution.ExecutionId, status, err)
return err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("[ERROR][%s] Failed reading agent '%s' body: %s", workflowExecution.ExecutionId, status, err)
return err
}
if resp.StatusCode != 200 {
log.Printf("[ERROR][%s] Failed sending self-request with '%s' for agent: %s", workflowExecution.ExecutionId, status, string(body))
return errors.New(fmt.Sprintf("No result in %s request for agent", status))
}
if status == "SUCCESS" || status == "FINISHED" || status == "FAILURE" || status == "ABORTED" {
// Check if this workflow is running in different env (not cloud)
fullExecution, err := GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
if err != nil {
log.Printf("[WARNING][%s] Failed getting full execution for AI Agent redeployment: %s", workflowExecution.ExecutionId, err)
} else {
// Get environment from the execution or from other workflow actions
agentEnvironment := ""
// First try: Check if any action in the workflow has an environment set
for _, action := range fullExecution.Workflow.Actions {
if len(action.Environment) > 0 {
agentEnvironment = action.Environment
break
}
}
// Fallback: Check execution environment
if len(agentEnvironment) == 0 && len(fullExecution.Workflow.ExecutionEnvironment) > 0 {
agentEnvironment = fullExecution.Workflow.ExecutionEnvironment
}
log.Printf("[DEBUG][%s] AI Agent finished. Detected environment: '%s'", workflowExecution.ExecutionId, agentEnvironment)
if strings.ToLower(agentEnvironment) != "cloud" && agentEnvironment != "" {
log.Printf("[INFO][%s] AI Agent finished (status: %s). Redeploying workflow to env '%s' with MAX priority",
workflowExecution.ExecutionId, status, agentEnvironment)
executionRequest := ExecutionRequest{
ExecutionId: fullExecution.ExecutionId,
WorkflowId: fullExecution.Workflow.ID,
Authorization: fullExecution.Authorization,
Environments: []string{agentEnvironment},
Priority: 11, // I'm assuming 11 is the max priority
}
parsedEnv := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(agentEnvironment, " ", "-"), "_", "-")), fullExecution.ExecutionOrg)
log.Printf("[INFO][%s] Redeploying workflow to queue: %s with priority %d", fullExecution.ExecutionId, parsedEnv, executionRequest.Priority)
// log.Printf("[DEBUG] AI Agent finished - REDEPLOYING workflow to Queue: '%s' (Priority: %d). Original Env: '%s', Org: '%s'", parsedEnv, executionRequest.Priority, agentEnvironment, fullExecution.ExecutionOrg)
err = SetWorkflowQueue(ctx, executionRequest, parsedEnv)
if err != nil {
log.Printf("[ERROR][%s] Failed redeploying workflow after AI Agent completion to env %s: %s", fullExecution.ExecutionId, parsedEnv, err)
} else {
log.Printf("[DEBUG][%s] Successfully redeployed workflow to %s after AI Agent completion", workflowExecution.ExecutionId, parsedEnv)
}
} else {
log.Printf("[DEBUG][%s] AI Agent finished (env: %s), no redeployment needed (cloud or empty env).", workflowExecution.ExecutionId, agentEnvironment)
}
}
}
return nil
}
// Handles the recursiveness of a stream result sent to the backend with an Agent Decision
// Used both in /streams from RunAgentDecisionAction() AND sendAgentActionSelfRequest()
// Further used for e.g. Question answers as to further guide the agent
// Also locally callable as we are doing in ai.go -> decision.Category == "standalone" -> ActionResult{}
func handleAgentDecisionStreamResult(workflowExecution WorkflowExecution, actionResult ActionResult) (*WorkflowExecution, bool, error) {
decisionIdSplit := strings.Split(actionResult.Status, "_")
decisionId := ""
if len(decisionIdSplit) > 1 {
if len(decisionIdSplit) == 2 {
decisionId = decisionIdSplit[1]
} else {
decisionId = strings.Join(decisionIdSplit[1:], "_")
}
}
if len(decisionId) == 0 {
log.Printf("[ERROR][%s] No decision ID found for node %s. This means we can't map the decision result in any way. Should we set the agent to FAILURE? RAW Status: %#v", actionResult.ExecutionId, actionResult.Action.ID, actionResult.Status)
return &workflowExecution, false, errors.New("Agent decision failed")
}
actionResult.Status = fmt.Sprintf("agent_%s", decisionId)
if debug {
log.Printf("[DEBUG][%s] Got decision ID '%s' for agent '%s'. Ref: %s", workflowExecution.ExecutionId, decisionId, actionResult.Action.ID, actionResult.Status)
}
ctx := context.Background()
foundActionResultIndex := -1
for actionIndex, result := range workflowExecution.Results {
if result.Action.ID == actionResult.Action.ID {
foundActionResultIndex = actionIndex
break
}
}
if foundActionResultIndex < 0 {
// In test mode, Singul doesn't create sub-executions, so we need to handle this gracefully
if os.Getenv("AGENT_TEST_MODE") == "true" {
log.Printf("[DEBUG][%s] AGENT_TEST_MODE: Action '%s' not found in results, creating placeholder", workflowExecution.ExecutionId, actionResult.Action.ID)
// Try to get the initial agent output from cache
ctx := context.Background()
actionCacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID)
placeholderResult := `{"status":"RUNNING","decisions":[]}`
cache, err := GetCache(ctx, actionCacheId)
if err == nil {
// Found cached agent output - use it!
cacheData := []byte(cache.([]uint8))
log.Printf("[DEBUG][%s] Found cached agent output for placeholder (size: %d bytes)", workflowExecution.ExecutionId, len(cacheData))
placeholderResult = string(cacheData)
} else {
log.Printf("[DEBUG][%s] No cached agent output found, using empty placeholder", workflowExecution.ExecutionId)
}
// Create a placeholder result for the agent action
placeholder := ActionResult{
Action: actionResult.Action,
ExecutionId: workflowExecution.ExecutionId,
Result: placeholderResult,
StartedAt: time.Now().Unix(),
CompletedAt: 0,
Status: "EXECUTING",
}
workflowExecution.Results = append(workflowExecution.Results, placeholder)
foundActionResultIndex = len(workflowExecution.Results) - 1
log.Printf("[DEBUG][%s] Created placeholder result at index %d", workflowExecution.ExecutionId, foundActionResultIndex)
} else {
log.Printf("[ERROR][%s] Action '%s' was NOT found with any result in the execution (yet)", workflowExecution.ExecutionId, actionResult.Action.ID)
return &workflowExecution, false, errors.New(fmt.Sprintf("ActionResultIndex: Agent node ID for decision ID %s not found", decisionId))
}
}
mappedResult := AgentOutput{}
//err := json.Unmarshal([]byte(actionResult.Result), &mappedResult)
err := json.Unmarshal([]byte(workflowExecution.Results[foundActionResultIndex].Result), &mappedResult)
if err != nil {
log.Printf("[ERROR][%s] Failed unmarshalling agent result: %s. Data: %s", workflowExecution.ExecutionId, err, actionResult.Result)
return &workflowExecution, false, err
}
// In test mode, if the placeholder has no decisions, we need to add the incoming decision
if os.Getenv("AGENT_TEST_MODE") == "true" && len(mappedResult.Decisions) == 0 {
log.Printf("[DEBUG][%s] AGENT_TEST_MODE: Placeholder has no decisions, parsing incoming decision", workflowExecution.ExecutionId)
// Parse the incoming decision from actionResult
incomingDecision := AgentDecision{}
err = json.Unmarshal([]byte(actionResult.Result), &incomingDecision)
if err != nil {
log.Printf("[ERROR][%s] Failed unmarshalling incoming decision: %s", workflowExecution.ExecutionId, err)
} else {
// Add the decision to the mapped result
mappedResult.Decisions = append(mappedResult.Decisions, incomingDecision)
mappedResult.Status = "RUNNING"
// Update the workflow execution result with the new decision
updatedResult, _ := json.Marshal(mappedResult)
workflowExecution.Results[foundActionResultIndex].Result = string(updatedResult)
log.Printf("[DEBUG][%s] Added decision %s to placeholder (total decisions: %d)", workflowExecution.ExecutionId, incomingDecision.RunDetails.Id, len(mappedResult.Decisions))
}
}
// FIXME: Need to check the current value from the workflowexecution here, instead of using the currently sent in decision
// 1. Get the current result for the action
// 2. Find the decision in there
decisionIdResultIndex := -1 // Index of the item in the decision list
decisionIndex := -1 // Assigned index to it by LLM
for resultDecisionIndex, resultDecision := range mappedResult.Decisions {
if resultDecision.RunDetails.Id == decisionId {
//log.Printf("[DEBUG][%s] Current decision (%s) status is '%s'", workflowExecution.ExecutionId, resultDecision.RunDetails.Id, resultDecision.RunDetails.Status)
decisionIdResultIndex = resultDecisionIndex
decisionIndex = resultDecision.I
break
}
}
if decisionIdResultIndex < 0 {
log.Printf("[ERROR][%s] Decision ID %s was not found. Skipping.", workflowExecution.ExecutionId, decisionId)
return &workflowExecution, false, errors.New(fmt.Sprintf("decisionIdResultIndex: Agent node ID for decision ID %s not found", decisionId))
}
if strings.Contains(actionResult.Result, decisionId) {
if debug {
log.Printf("[DEBUG][%s] Mapping decision result (stream result) for decision ID '%s'", workflowExecution.ExecutionId, decisionId)
}
newDecision := AgentDecision{}
//err = json.Unmarshal([]byte(actionResult.Result), &mappedResult.Decisions[decisionIdResultIndex])
err = json.Unmarshal([]byte(actionResult.Result), &newDecision)
if err != nil {
log.Printf("[ERROR][%s] Failed unmarshalling agent decision result for decision ID '%s': %s. Data: %s", workflowExecution.ExecutionId, decisionId, err, actionResult.Result)
} else {
if newDecision.Action == "answer" {
if debug {
log.Printf("[DEBUG] Auto-finishing 'answer' decision for decision ID '%s'", decisionId)
}
newDecision.RunDetails.Status = "FINISHED"
mappedResult.Decisions[decisionIdResultIndex] = newDecision
}
if newDecision.RunDetails.Status != "" && newDecision.RunDetails.Id != "" && (newDecision.RunDetails.Status != mappedResult.Decisions[decisionIdResultIndex].RunDetails.Status || newDecision.RunDetails.StartedAt != mappedResult.Decisions[decisionIdResultIndex].RunDetails.StartedAt || newDecision.RunDetails.CompletedAt != mappedResult.Decisions[decisionIdResultIndex].RunDetails.CompletedAt) {
log.Printf("[DEBUG][%s] Updating decision ID '%s' with new status '%s' (old: '%s')", workflowExecution.ExecutionId, decisionId, newDecision.RunDetails.Status, mappedResult.Decisions[decisionIdResultIndex].RunDetails.Status)
mappedResult.Decisions[decisionIdResultIndex] = newDecision
// Set cache for action, agent & decision here as well (?)
decisionCacheId := fmt.Sprintf("agent-%s-%s", workflowExecution.ExecutionId, newDecision.RunDetails.Id)
err = SetCache(context.Background(), decisionCacheId, []byte(actionResult.Result), 300)
if err != nil {
log.Printf("[ERROR][%s] Failed setting cache for decision ID '%s': %s", workflowExecution.ExecutionId, decisionId, err)
}
}
}
// Update cache here as well
//decisionCache := fmt.Sprintf("%s_%s_decision_%s", workflowExecution.ExecutionId, actionResult.Action.ID, decisionId)
}
if debug {
log.Printf("[DEBUG][%s] Action '%s' AND decision ID '%s' (%d). Decision Index: %d. Continue decisionmaking!", workflowExecution.ExecutionId, actionResult.Action.ID, decisionId, decisionIdResultIndex, decisionIndex)
}
if mappedResult.Decisions[decisionIdResultIndex].RunDetails.Status == "FAILURE" || mappedResult.Decisions[decisionIdResultIndex].RunDetails.Status == "ABORTED" {
if debug {
log.Printf("[DEBUG] Auto-failing agent due to decision ID '%s' being in status '%s'", decisionId, mappedResult.Decisions[decisionIdResultIndex].RunDetails.Status)
}
//go sendAgentActionSelfRequest("FAILURE", workflowExecution, workflowExecution.Results[foundActionResultIndex])
//return &workflowExecution, false, nil
}
//mappedResult.Decisions[decisionIdResultIndex] = actionResult.Result
// Find next action
allFinishedDecisions := []string{}
for decisionId, curDecision := range mappedResult.Decisions {
if curDecision.RunDetails.Status == "FINISHED" || curDecision.RunDetails.Status == "IGNORED" {
allFinishedDecisions = append(allFinishedDecisions, curDecision.RunDetails.Id)
} else if curDecision.RunDetails.Status == "FAILURE" {
if debug {
log.Printf("[DEBUG] Treating decision ID '%s' as finished due to FAILURE status", curDecision.RunDetails.Id)
}
allFinishedDecisions = append(allFinishedDecisions, curDecision.RunDetails.Id)
}
if curDecision.I <= decisionIndex {
continue
}
foundDecisions := []AgentDecision{}
parentIndex := curDecision.I - 1
for _, subDecision := range mappedResult.Decisions {
if subDecision.I == parentIndex {
foundDecisions = append(foundDecisions, subDecision)
}
}
if len(foundDecisions) == 0 {
continue
}
finishedDecisions := []string{}
failedDecisions := []string{}
for _, foundDecision := range foundDecisions {
if foundDecision.RunDetails.Status == "RUNNING" {
continue
} else if foundDecision.RunDetails.Status == "FAILED" {
failedDecisions = append(failedDecisions, foundDecision.RunDetails.Id)
} else if foundDecision.RunDetails.Status == "FINISHED" || foundDecision.RunDetails.Status == "IGNORED" {
finishedDecisions = append(finishedDecisions, foundDecision.RunDetails.Id)
} else {
log.Printf("[ERROR][%s] No handler for run status %s", workflowExecution.ExecutionId, foundDecision.RunDetails.Status)
}
}
// FIXME: Set the status of the node to failed
if len(failedDecisions) > 0 {
log.Printf("[WARNING][%s] Failed decision found. Should exit out agent %d. It should have exited before this point.", workflowExecution.ExecutionId, decisionId)
//go sendAgentActionSelfRequest("FAILURE", workflowExecution, workflowExecution.Results[foundActionResultIndex])
//break
}
if len(foundDecisions) == len(finishedDecisions) {
mappedResult.Decisions[decisionId].RunDetails.Status = "RUNNING"
mappedResult.Decisions[decisionId].RunDetails.StartedAt = time.Now().UnixMilli()
go RunAgentDecisionAction(workflowExecution, mappedResult, curDecision)
}
}
log.Printf("[INFO] TOTAL AGENT DECISIONS: %#v, FINISHED DECISIONS: %#v. Finished decision IDs: %#v", len(mappedResult.Decisions), len(allFinishedDecisions), allFinishedDecisions)
// FIXME: How do we handle 3rd party memory sources?
// This uses the built-in datastore mechanism so that the user
// can see and modify stuff themself as well.
if mappedResult.Memory == "shuffle_db" {
requestKey := fmt.Sprintf("chat_%s_%s", actionResult.ExecutionId, actionResult.Action.ID)
if debug {
log.Printf("[DEBUG] Getting agent chat history: %s", requestKey)
}
agentRequestMemory, err := GetDatastoreKey(ctx, requestKey, "agent_requests")
if err != nil {
log.Printf("[ERROR][%s] Failed to find request memory for updates", actionResult.ExecutionId)
} else {
if len(agentRequestMemory.Value) > 0 {
log.Printf("[DEBUG] Found cache memory in shuffle datastore: \n\n%s", agentRequestMemory.Value)
} else {
log.Printf("[DEBUG] No agent cache memory for key %s", requestKey)
}
}
}
if len(allFinishedDecisions) == len(mappedResult.Decisions) {
// Handle agent decisionmaking. Use the same
log.Printf("[INFO][%s] With the agent being finished, we are asking it whether it would like to do anything else", workflowExecution.ExecutionId)
var originalAction Action
if foundActionResultIndex >= 0 && foundActionResultIndex < len(workflowExecution.Results) {
originalAction = workflowExecution.Results[foundActionResultIndex].Action
} else {
// Fallback in case of an issue
originalAction = actionResult.Action
}
callerName := "handleAgentDecisionStreamResult"
returnAction, err := HandleAiAgentExecutionStart(workflowExecution, originalAction, true, callerName)
if err != nil {
log.Printf("[ERROR][%s] Failed handling agent execution start: %s", workflowExecution.ExecutionId, err)
}
_ = returnAction
//go sendAgentActionSelfRequest("SUCCESS", workflowExecution, workflowExecution.Results[foundActionResultIndex])
return &workflowExecution, false, nil
}
return &workflowExecution, true, nil
}
// Updateparam is a check to see if the execution should be continuously validated
func ParsedExecutionResult(ctx context.Context, workflowExecution WorkflowExecution, actionResult ActionResult, updateParam bool, retries int64) (*WorkflowExecution, bool, error) {
var err error
if actionResult.Action.ID == "" && actionResult.Action.Name == "" {
// Can we find it based on label?
//log.Printf("\n\n[ERROR][%s] Failed handling EMPTY action %#v (ParsedExecutionResult). Usually ONLY happens during worker run that sets everything?\n\n", workflowExecution.ExecutionId, actionResult)
return &workflowExecution, true, nil
}
// 1. CHECK cache if it happened in another?
// 2. Set cache
// 3. Find executed without a result
// 4. Ensure the result is NOT set when running an action)
actionResult = FixActionResultOutput(actionResult)
actionResult.Sanitized = false
actionCacheId := fmt.Sprintf("%s_%s_result", actionResult.ExecutionId, actionResult.Action.ID)
// Done elsewhere
setCache := true
if actionResult.Action.AppName == "shuffle-subflow" {
// Verifying if the userinput should be sent properly or not
if actionResult.Action.Name == "run_userinput" && actionResult.Status != "SKIPPED" {
// log.Printf("\n\n[INFO] Inside userinput default return! Return data: %s", actionResult.Result)
actionResult.Status = "WAITING"
actionResult.CompletedAt = time.Now().Unix() * 1000
if strings.Contains(actionResult.Result, "\"success\":") {
//log.Printf("Found success in result. Now verifying if the workflow should just continue or not")
type SubflowMapping struct {
Success bool `json:"success"`
}
var subflowData SubflowMapping
err := json.Unmarshal([]byte(actionResult.Result), &subflowData)
if err == nil && subflowData.Success == false {
log.Printf("[INFO][%s] Userinput subflow failed. Should abort workflow or continue execution by default?", actionResult.ExecutionId)
} else {
log.Printf("[INFO][%s] Userinput subflow succeeded. Should continue execution by default?", actionResult.ExecutionId)
// FIXME:
// 1. What should happen on cloud?
// 2. What should happen if on backend with NON cloud env?
// 3. What should happen if inside Worker?
// 4. What if Swarm?
setWorkflow := false
if strings.ToLower(actionResult.Action.Environment) != "cloud" {
if project.Environment == "worker" {
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" {
//log.Printf("\n\n\n[DEBUG] MODIFYING workflow based on User Input as we are in swarm\n\n\n")
workflowExecution.Status = "WAITING"
workflowExecution.Results = append(workflowExecution.Results, actionResult)
setWorkflow = true
} else {
log.Printf("\n\n\n[DEBUG] NOT modifying workflow based on User Input as we are in worker\n\n\n")
}
} else {
// Find the waiting node and change it to this result
workflowExecution.Status = "WAITING"
workflowExecution.Results = append(workflowExecution.Results, actionResult)
setWorkflow = true
}
}
if setWorkflow {
// Set with database saving
err = SetWorkflowExecution(ctx, workflowExecution, true)
if err != nil {
log.Printf("[ERROR][%s] Failed setting workflow execution during user input return onprem~: %s", workflowExecution.ExecutionId, err)
}
}
if strings.Contains(actionResult.Result, "\"execution_id\":") && strings.Contains(actionResult.Result, "\"authorization\":") {
log.Printf("\n\n[DEBUG][%s] Found execution_id and authorization in result. Now verifying if the workflow should just continue or not\n\n", actionResult.ExecutionId)
return &workflowExecution, false, errors.New("User Input")
}
}
}
// Finding the waiting node and changing it to this result
foundWaiting := false
for resultIndex, result := range workflowExecution.Results {
if result.Action.ID != actionResult.Action.ID {
continue
}
workflowExecution.Results[resultIndex].Result = actionResult.Result
// Updating cache for the result to always use the latest
//actionResultBody, err := json.Marshal(workflowExecution.Results[resultIndex].Result)
actionResultBody, err := json.Marshal(actionResult)
if err == nil {
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID)
err = SetCache(ctx, cacheId, actionResultBody, 35)
if err != nil {
log.Printf("[WARNING] Couldn't find in fix exec %s (2): %s", cacheId, err)
continue
}
}
foundWaiting = true
break
}
if !foundWaiting {
workflowExecution.Results = append(workflowExecution.Results, actionResult)
actionResultBody, err := json.Marshal(actionResult)
if err == nil {
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID)
err = SetCache(ctx, cacheId, actionResultBody, 35)
if err != nil {
log.Printf("[ERROR][%s] Failed to update cache for %s", workflowExecution.ExecutionId, cacheId)
}
}
}
err = SetWorkflowExecution(ctx, workflowExecution, true)
if err != nil {
log.Printf("[ERROR][%s] Failed setting workflow execution during user input return: %s", workflowExecution.ExecutionId, err)
}
return &workflowExecution, true, nil
} else {
// Should NOT run with all this if the action is SKIPPED
// Cache when SKIPPED - this is to handle the case where the subflow is skipped (condition) and the result is not set
if actionResult.Status == "SKIPPED" {
setCache = true
} else {
for _, param := range actionResult.Action.Parameters {
if param.Name == "check_result" {
if param.Value == "true" {
setCache = false
}
break
}
}
}
if !setCache {
var subflowData SubflowData
jsonerr := json.Unmarshal([]byte(actionResult.Result), &subflowData)
if jsonerr == nil && len(subflowData.Result) == 0 && !strings.Contains(actionResult.Result, "\"result\"") {
setCache = false
} else {
setCache = true
}
}
}
//log.Printf("[DEBUG] Skipping setcache for subflow? SetCache: %t", setCache)
} else if actionResult.Action.AppName == "AI Agent" || actionResult.Action.AppName == "Shuffle Agent" {
if strings.HasPrefix(actionResult.Status, "agent_") {
if debug {
log.Printf("[DEBUG] Got AI Agent response - STATUS: %#v, resp: %#v.", actionResult.Status, actionResult.Result)
}
return handleAgentDecisionStreamResult(workflowExecution, actionResult)
}
}
if setCache {
go RunExecutionTranslation(ctx, actionResult)
actionResultBody, err := json.Marshal(actionResult)
if err == nil {
err = SetCache(ctx, actionCacheId, actionResultBody, 35)
if err != nil {
//log.Printf("\n\n\n[ERROR] Failed setting cache for action in parsed exec results %s: %s\n\n", actionCacheId, err)
}
} else {
log.Printf("[ERROR] Failed marshalling result and put it in cache.")
}
} else {
//log.Printf("[WARNING] Skipping cache for %s", actionResult.Action.Name)
}
skipExecutionCount := false
if workflowExecution.Status == "FINISHED" {
skipExecutionCount = true
}
dbSave := false
startAction, extra, children, parents, visited, executed, nextActions, environments := GetExecutionVariables(ctx, workflowExecution.ExecutionId)
// Shitty workaround as it may be missing it at times
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == actionResult.Action.ID {
//log.Printf("HAS EXEC VARIABLE: %s", action.ExecutionVariable)
actionResult.Action.ExecutionVariable = action.ExecutionVariable
break
}
}
newResult := FixBadJsonBody([]byte(actionResult.Result))
actionResult.Result = string(newResult)
if len(actionResult.Action.ExecutionVariable.Name) > 0 && (actionResult.Status == "SUCCESS" || actionResult.Status == "FINISHED") {
// Should just check the first bytes for this, as it should be at the start if it's a failure with the individual action itself
// This is finicky, but it's the easiest fix for this
if setExecutionVariable(actionResult) {
if debug {
log.Printf("[DEBUG][%s] Updating exec variable '%s' with new value from node '%s' of length %d (2)", workflowExecution.ExecutionId, actionResult.Action.ExecutionVariable.Name, actionResult.Action.Label, len(actionResult.Result))
}
if len(workflowExecution.Results) > 0 {
// Should this be used?
// lastResult := workflowExecution.Results[len(workflowExecution.Results)-1].Result
}
actionResult.Action.ExecutionVariable.Value = actionResult.Result
foundIndex := -1
for i, executionVariable := range workflowExecution.ExecutionVariables {
if executionVariable.Name == actionResult.Action.ExecutionVariable.Name {
foundIndex = i
break
}
}
if foundIndex >= 0 {
workflowExecution.ExecutionVariables[foundIndex] = actionResult.Action.ExecutionVariable
} else {
workflowExecution.ExecutionVariables = append(workflowExecution.ExecutionVariables, actionResult.Action.ExecutionVariable)
}
} else {
log.Printf("[DEBUG] NOT updating exec variable %s with new value of length %d. Check previous errors, or if action was successful (success: true)", actionResult.Action.ExecutionVariable.Name, len(actionResult.Result))
}
}
if workflowExecution.Workflow.Configuration.SkipNotifications == false && actionResult.Status == "SUCCESS" && strings.Contains(actionResult.Result, "\"success\":") && strings.Contains(actionResult.Result, "\"status\":") {
type resultMapping struct {
Success bool `json:"success"`
Status int `json:"status"`
}
var mapping resultMapping
err := json.Unmarshal([]byte(actionResult.Result), &mapping)
if err == nil && mapping.Success == true && mapping.Status >= 300 {
//log.Printf("\n\n[DEBUG] Setting status to failure as it's a success with status code %d\n\n", mapping.Status)
parsedDescription := fmt.Sprintf("Bad status code in action %s: %d. This shows up if status is >= 300", actionResult.Action.Name, mapping.Status)
if mapping.Status == 404 {
parsedDescription = fmt.Sprintf("404 not found for action %s. Check if the URL is correct, and that the data it is trying to retrieve exists.", actionResult.Action.Name)
}
if mapping.Status == 401 {
parsedDescription = fmt.Sprintf("401 unauthorized for action %s. Make sure your credentials are correct.", actionResult.Action.Name)
}
if mapping.Status == 403 {
parsedDescription = fmt.Sprintf("403 forbidden for action %s. Make sure the account you are using has access to the resource.", actionResult.Action.Name)
}
// Send notification for it
err := CreateOrgNotification(
ctx,
fmt.Sprintf("Bad Status code in Workflow %s: %d", workflowExecution.Workflow.Name, mapping.Status),
parsedDescription,
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, actionResult.Action.ID),
workflowExecution.ExecutionOrg,
true,
"CRITICAL",
"workflow_execution",
)
workflowExecution.NotificationsCreated++
if err != nil {
log.Printf("[ERROR] Failed making org notification (1): %s", err)
}
}
}
actionResult.Action = Action{
AppName: actionResult.Action.AppName,
AppVersion: actionResult.Action.AppVersion,
Label: actionResult.Action.Label,
Name: actionResult.Action.Name,
ID: actionResult.Action.ID,
Parameters: actionResult.Action.Parameters,
ExecutionVariable: actionResult.Action.ExecutionVariable,
}
// Cleaning up result authentication
notificationSent := false
for paramIndex, param := range actionResult.Action.Parameters {
if param.Configuration {
//log.Printf("[INFO] Deleting param %s (auth)", param.Name)
actionResult.Action.Parameters[paramIndex].Value = ""
}
if param.Name == "liquid_syntax_error" && !notificationSent {
// Send notification for it
err := CreateOrgNotification(
ctx,
fmt.Sprintf("Liquid Syntax Error in Workflow %s", workflowExecution.Workflow.Name),
fmt.Sprintf("Node %s in Workflow %s was found to have a Liquid Syntax Error. Click to investigate", actionResult.Action.Label, workflowExecution.Workflow.Name),
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, actionResult.Action.ID),
workflowExecution.ExecutionOrg,
true,
"CRITICAL",
"liquid_syntax",
)
workflowExecution.NotificationsCreated++
if err == nil {
notificationSent = true
} else {
log.Printf("[ERROR] Failed making org notification (2): %s", err)
}
}
}
// Used for testing subflow shit
//if strings.Contains(actionResult.Action.Label, "Shuffle Workflow_30") {
// log.Printf("RESULT FOR %s: %s", actionResult.Action.Label, actionResult.Result)
// if !strings.Contains(actionResult.Result, "\"result\"") {
// log.Printf("NO RESULT - RETURNING!")
// return &workflowExecution, false, nil
// }
//}
// Fills in data from subflows, whether they're loops or not
// Update: handling this farther down the function
//log.Printf("[DEBUG] STATUS OF %s: %s", actionResult.Action.AppName, actionResult.Status)
if actionResult.Status == "SUCCESS" && actionResult.Action.AppName == "shuffle-subflow" {
dbSave = true
}
if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" {
IncrementCache(ctx, workflowExecution.ExecutionOrg, "app_executions_failed")
if workflowExecution.Workflow.Configuration.SkipNotifications == false {
// Add an else for HTTP request errors with success "false"
// These could be "silent" issues
if actionResult.Status == "FAILURE" && workflowExecution.Workflow.Hidden == false {
log.Printf("[DEBUG] Result is %s for %s (%s). Making notification.", actionResult.Status, actionResult.Action.Label, actionResult.Action.ID)
err := CreateOrgNotification(
ctx,
fmt.Sprintf("Error in Workflow %s", workflowExecution.Workflow.Name),
fmt.Sprintf("Node %s in Workflow %s was found to have an error. Click to investigate", actionResult.Action.Label, workflowExecution.Workflow.Name),
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, actionResult.Action.ID),
workflowExecution.ExecutionOrg,
true,
"CRITICAL",
"action_failure",
)
workflowExecution.NotificationsCreated++
if err != nil {
log.Printf("[ERROR] Failed making org notification (3): %s", err)
}
}
}
newResults := []ActionResult{}
childNodes := []string{}
if workflowExecution.Workflow.Configuration.ExitOnError {
// Find underlying nodes and add them
log.Printf("[WARNING][%s] Actionresult is %s for node %s (%s). Should set workflowExecution and exit all running functions", workflowExecution.ExecutionId, actionResult.Status, actionResult.Action.Label, actionResult.Action.ID)
workflowExecution.Status = actionResult.Status
workflowExecution.LastNode = actionResult.Action.ID
if len(workflowExecution.Workflow.DefaultReturnValue) > 0 {
workflowExecution.Result = workflowExecution.Workflow.DefaultReturnValue
}
IncrementCache(ctx, workflowExecution.ExecutionOrg, "workflow_executions_failed")
} else {
log.Printf("[WARNING][%s] Actionresult is %s for node %s. Continuing anyway because of workflow configuration.", workflowExecution.ExecutionId, actionResult.Status, actionResult.Action.ID)
// Finds ALL childnodes to set them to SKIPPED
// Remove duplicates
childNodes = FindChildNodes(workflowExecution.Workflow, actionResult.Action.ID, []string{}, []string{})
//log.Printf("[DEBUG][%s] FOUND %d CHILDNODES\n\n", workflowExecution.ExecutionId, len(childNodes))
for _, nodeId := range childNodes {
if debug {
log.Printf("[DEBUG][%s] Checking if node %s is already in results", workflowExecution.ExecutionId, nodeId)
}
if nodeId == actionResult.Action.ID {
log.Printf("[DEBUG][%s] Skipping marking node %s (%s) as anything", workflowExecution.ExecutionId, nodeId, actionResult.Action.Label)
continue
}
// 1. Find the action itself
// 2. Create an actionresult
curAction := Action{ID: ""}
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == nodeId {
curAction = action
if debug {
log.Printf("[DEBUG][%s] Found action %s (%s) for node %s", workflowExecution.ExecutionId, action.Label, action.ID, nodeId)
}
break
}
}
if debug {
log.Printf("[DEBUG][%s] Found action with ID: %s", workflowExecution.ExecutionId, curAction.ID)
}
isTrigger := false
if len(curAction.ID) == 0 {
for _, trigger := range workflowExecution.Workflow.Triggers {
//log.Printf("%s : %s", trigger.ID, nodeId)
if trigger.ID == nodeId {
isTrigger = true
name := "shuffle-subflow"
curAction = Action{
AppName: name,
AppVersion: trigger.AppVersion,
Label: trigger.Label,
Name: trigger.Name,
ID: trigger.ID,
}
//log.Printf("SET NODE!!")
break
}
}
if len(curAction.ID) == 0 {
//log.Printf("Couldn't find subnode %s", nodeId)
log.Printf("[WARNING][%s] Couldn't find subnode %s. Forgetting about it", workflowExecution.ExecutionId, nodeId)
continue
}
}
resultExists := false
for _, result := range workflowExecution.Results {
//log.Printf("[DEBUG][%s] Checking if result %s (%s) exists in results", workflowExecution.ExecutionId, result.Action.Label, result.Action.ID)
if result.Action.ID == curAction.ID {
resultExists = true
break
}
}
if !resultExists {
// Check parents are done here. Only add it IF all parents are skipped
skipNodeAdd := false
for _, branch := range workflowExecution.Workflow.Branches {
if branch.DestinationID == nodeId && !isTrigger {
// If the branch's source node is NOT in childNodes, it's not a skipped parent
// Checking if parent is a trigger
parentTrigger := false
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.ID == branch.SourceID {
if trigger.AppName != "User Input" && trigger.AppName != "Shuffle Workflow" {
//log.Printf("[DEBUG][%s] Parent %s (%s) is a trigger. Continuing..", workflowExecution.ExecutionId, branch.SourceID, curAction.Label)
parentTrigger = true
}
}
}
if parentTrigger {
if debug {
log.Printf("[DEBUG][%s] Parent %s (of child %s) is a trigger. Continuing..", workflowExecution.ExecutionId, branch.SourceID, nodeId)
}
continue
}
//log.Printf("[DEBUG][%s] Parent %s (of child %s) is NOT a trigger. Continuing..", workflowExecution.ExecutionId, branch.SourceID, nodeId)
sourceNodeFound := false
for _, item := range childNodes {
if item == branch.SourceID {
if debug {
log.Printf("[DEBUG][%s] Found source node %s (%s) for node %s", workflowExecution.ExecutionId, branch.SourceID, curAction.Label, nodeId)
}
sourceNodeFound = true
break
}
}
if debug {
log.Printf("[DEBUG][%s] sourceNodeFound: %t for node %s", workflowExecution.ExecutionId, sourceNodeFound, nodeId)
}
if !sourceNodeFound {
// FIXME: Shouldn't add skip for child nodes of these nodes. Check if this node is parent of upcoming nodes.
//log.Printf("\n\n NOT setting node %s to SKIPPED", nodeId)
skipNodeAdd = true
if !ArrayContains(visited, nodeId) && !ArrayContains(executed, nodeId) {
nextActions = append(nextActions, nodeId)
log.Printf("[INFO] SHOULD EXECUTE NODE %s. Next actions: %s", nodeId, nextActions)
}
break
}
}
}
if !skipNodeAdd {
newResult := ActionResult{
Action: curAction,
ExecutionId: actionResult.ExecutionId,
Authorization: actionResult.Authorization,
Result: `{"success": false, "reason": "Skipped because of previous node - 2"}`,
StartedAt: 0,
CompletedAt: 0,
Status: "SKIPPED",
}
newResults = append(newResults, newResult)
visited = append(visited, curAction.ID)
executed = append(executed, curAction.ID)
UpdateExecutionVariables(ctx, workflowExecution.ExecutionId, startAction, children, parents, visited, executed, nextActions, environments, extra)
} else {
// log.Printf("\n\nNOT adding %s as skipaction - should add to execute?", nodeId)
//var visited []string
//var executed []string
//var nextActions []string
log.Printf("[DEBUG][%s] Not adding %s - %s as a skipaction.", workflowExecution.ExecutionId, curAction.ID, nodeId)
}
}
}
}
// Cleans up aborted, and always gives a result
lastResult := ""
// type ActionResult struct {
for _, result := range workflowExecution.Results {
if debug {
log.Printf("[DEBUG][%s] Checking result '%s' (%s) with status %s", workflowExecution.ExecutionId, result.Action.Label, result.Action.ID, result.Status)
}
if actionResult.Action.ID == result.Action.ID {
continue
}
if result.Status == "EXECUTING" {
result.Status = actionResult.Status
result.Result = "Aborted because of error in another node (2)"
}
if len(result.Result) > 0 && result.Status == "SUCCESS" {
lastResult = result.Result
}
newResults = append(newResults, result)
}
if workflowExecution.LastNode == "" {
workflowExecution.LastNode = actionResult.Action.ID
}
workflowExecution.Result = lastResult
workflowExecution.Results = newResults
}
if actionResult.Status == "SKIPPED" {
//childNodes := FindChildNodes(workflowExecution, actionResult.Action.ID)
// See if it can even find it in here for skipped?
//log.Printf("childnodes of %s (%s): %d: %s", actionResult.Action.Label, actionResult.Action.Id, len(childNodes), childNodes)
//FIXME: Should this run and fix all nodes,
// or should it send them in as new SKIPs? Should we only handle DIRECT
// children? I wonder.
//log.Printf("\n\n\n[DEBUG] FROM %s - FOUND childnode %s %s (%s). exists: %s\n\n\n", actionResult.Action.Label, curAction.ID, curAction.Name, curAction.Label, resultExists)
// FIXME: Add triggers
for _, branch := range workflowExecution.Workflow.Branches {
if branch.SourceID != actionResult.Action.ID {
continue
}
// Find the target & check if it has more branches. If it does, and they're not finished - continue
foundAction := Action{}
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == branch.DestinationID {
foundAction = action
break
}
}
if len(foundAction.ID) == 0 {
for _, trigger := range workflowExecution.Workflow.Triggers {
//if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
if trigger.ID == branch.DestinationID {
foundAction = Action{
ID: trigger.ID,
AppName: trigger.AppName,
Name: trigger.AppName,
Label: trigger.Label,
}
if trigger.AppName == "Shuffle Workflow" {
foundAction.AppName = "shuffle-subflow"
}
break
}
}
if len(foundAction.ID) == 0 {
continue
}
}
// FIXME: Debug logs necessary to understand how workflows finish?
if debug {
log.Printf("[DEBUG][%s] Found that %s (%s) should be skipped? Should check if it has more parents. If not, send in a skip", workflowExecution.ExecutionId, foundAction.Label, foundAction.AppName)
}
foundCount := 0
skippedBranches := []string{}
for _, checkBranch := range workflowExecution.Workflow.Branches {
if checkBranch.DestinationID == foundAction.ID {
foundCount += 1
// Check if they're all skipped or not
if checkBranch.SourceID == actionResult.Action.ID {
skippedBranches = append(skippedBranches, checkBranch.SourceID)
continue
}
// Not found = not counted yet
for _, res := range workflowExecution.Results {
if res.Action.ID == checkBranch.SourceID && res.Status != "SUCCESS" && res.Status != "FINISHED" {
skippedBranches = append(skippedBranches, checkBranch.SourceID)
break
}
}
}
}
skippedCount := len(skippedBranches)
//log.Printf("[DEBUG][%s] Found %d branch(es) for %s. %d skipped. If equal, make the node skipped. SKIPPED: %s", workflowExecution.ExecutionId, foundCount, foundAction.Label, skippedCount, skippedBranches)
if foundCount == skippedCount {
found := false
for _, res := range workflowExecution.Results {
if res.Action.ID == foundAction.ID {
found = true
}
}
// Send only IF it's not the startnode
//if !found {
if !found && foundAction.ID != workflowExecution.Start {
newResult := ActionResult{
Action: foundAction,
ExecutionId: actionResult.ExecutionId,
Authorization: actionResult.Authorization,
Result: fmt.Sprintf(`{"success": false, "reason": "Skipped because of previous node (%s) - 1"}`, actionResult.Action.Label),
StartedAt: 0,
CompletedAt: 0,
Status: "SKIPPED",
}
resultData, err := json.Marshal(newResult)
if err != nil {
log.Printf("[ERROR] Failed skipping action")
continue
}
streamUrl := fmt.Sprintf("http://localhost:5001/api/v1/streams")
if project.Environment == "cloud" {
streamUrl = fmt.Sprintf("https://shuffler.io/api/v1/streams")
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
streamUrl = fmt.Sprintf("https://%s.%s.r.appspot.com/api/v1/streams", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
streamUrl = fmt.Sprintf("%s/api/v1/streams", os.Getenv("SHUFFLE_CLOUDRUN_URL"))
}
} else {
if len(os.Getenv("WORKER_HOSTNAME")) > 0 {
streamUrl = fmt.Sprintf("http://%s:33333/api/v1/streams", os.Getenv("WORKER_HOSTNAME"))
}
if os.Getenv("SHUFFLE_OPTIMIZED") == "true" && len(os.Getenv("WORKER_PORT")) > 0 {
streamUrl = fmt.Sprintf("http://localhost:%s/api/v1/streams", os.Getenv("WORKER_PORT"))
} else if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
streamUrl = fmt.Sprintf("http://localhost:33333/api/v1/streams")
} else {
// Does this fuck it up? This should only run
// if the worker is NON OPTIMIZED. Problem:
// The worker needs to talk back to itself.
if len(os.Getenv("BASE_URL")) > 0 {
streamUrl = fmt.Sprintf("%s/api/v1/streams", os.Getenv("BASE_URL"))
}
}
}
//log.Printf("[DEBUG] Sending skip for action %s (%s) to URL %s", foundAction.Label, foundAction.AppName, streamUrl)
req, err := http.NewRequest(
"POST",
streamUrl,
bytes.NewBuffer([]byte(resultData)),
)
if err != nil {
log.Printf("[ERROR] Error building SKIPPED request (%s): %s", foundAction.Label, err)
continue
}
client := &http.Client{}
newresp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Error running SKIPPED request (%s): %s", foundAction.Label, err)
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, foundAction.ID)
go SetCache(context.Background(), cacheId, resultData, 35)
continue
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading body when running SKIPPED request (%s): %s", foundAction.Label, err)
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, foundAction.ID)
go SetCache(context.Background(), cacheId, resultData, 35)
continue
}
//log.Printf("[DEBUG] Skipped body return from %s (%d): %s", streamUrl, newresp.StatusCode, string(body))
if strings.Contains(string(body), "already finished") {
log.Printf("[WARNING] Data couldn't be re-inputted for %s.", foundAction.Label)
// DONT CHANGE THE ERROR OUTPUT HERE
return &workflowExecution, true, errors.New(fmt.Sprintf("Workflow has already been ran with label %s. Raw: %s", foundAction.Label, string(body)))
}
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, foundAction.ID)
go SetCache(context.Background(), cacheId, resultData, 35)
}
}
}
}
// Related to notifications
if actionResult.Status == "SUCCESS" && workflowExecution.Workflow.Configuration.SkipNotifications == false {
// Marshal default failures
resultCheck := ResultChecker{}
err = json.Unmarshal([]byte(actionResult.Result), &resultCheck)
if err == nil {
//log.Printf("\n\n[WARNING] Unmarshal success in workflow %s! Trying to check for success. Success: %#v\n\n", workflowExecution.Workflow.Name, resultCheck.Success)
if strings.Contains(strings.Replace(actionResult.Result, " ", "", -1), `"success":false`) && resultCheck.Success == false && workflowExecution.Workflow.Hidden == false {
description := fmt.Sprintf("Node '%s' in Workflow '%s' failed silently. Failure Reason: %s", actionResult.Action.Label, workflowExecution.Workflow.Name, resultCheck.Reason)
if len(resultCheck.Reason) == 0 {
description = fmt.Sprintf("Node '%s' in Workflow '%s' failed silently. Check the workflow run for more details.", actionResult.Action.Label, workflowExecution.Workflow.Name)
}
err = CreateOrgNotification(
ctx,
fmt.Sprintf("Potential error in Workflow '%s'", workflowExecution.Workflow.Name),
description,
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, actionResult.Action.ID),
workflowExecution.ExecutionOrg,
true,
"HIGH",
"workflow_silent_failure",
)
workflowExecution.NotificationsCreated++
if err != nil {
log.Printf("[ERROR] Failed making org notification for %s (4): %s", workflowExecution.ExecutionOrg, err)
}
}
} else {
//log.Printf("[ERROR] Failed unmarshaling result into resultChecker (%s): %s", err, actionResult)
}
//log.Printf("[DEBUG] Ran marshal on silent failure")
}
// Handles notification handling for data coming back from apps
for _, param := range actionResult.Action.Parameters {
//actionResult.NotificationsCreated += 1
if strings.HasPrefix(strings.ToLower(param.Name), "shuffle") && strings.Contains(param.Name, "error") {
workflowExecution.NotificationsCreated += 1
CreateOrgNotification(
ctx,
fmt.Sprintf("App error for node %s in Workflow %s: %s", actionResult.Action.Label, workflowExecution.Workflow.Name, param.Name),
fmt.Sprintf("The node %s (%s) in workflow %s (%s) had the error: '%s' based on error '%s'", actionResult.Action.Label, actionResult.Action.ID, workflowExecution.Workflow.Name, workflowExecution.Workflow.ID, param.Value, param.Name),
fmt.Sprintf("/workflows/%s?execution_id=%s&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, actionResult.Action.ID),
workflowExecution.ExecutionOrg,
true,
"CRITICAL",
"app_error",
)
}
}
// FIXME rebuild to be like this or something
// workflowExecution/ExecutionId/Nodes/NodeId
// Find the appropriate action
if len(workflowExecution.Results) > 0 {
// FIXME
skip := false
found := false
outerindex := 0
for index, item := range workflowExecution.Results {
if item.Action.ID == actionResult.Action.ID {
found = true
if item.Status == actionResult.Status {
skip = true
}
outerindex = index
break
}
}
if skip {
//log.Printf("[DEBUG] Both results are %s. Skipping this node", item.Status)
} else if found {
// If result exists and execution variable exists, update execution value
//log.Printf("Exec var backend: %s", workflowExecution.Results[outerindex].Action.ExecutionVariable.Name)
actionVarName := workflowExecution.Results[outerindex].Action.ExecutionVariable.Name
// Finds potential execution arguments
if len(actionVarName) > 0 {
//log.Printf("EXECUTION VARIABLE LOCAL: %s", actionVarName)
for index, execvar := range workflowExecution.ExecutionVariables {
if execvar.Name == actionVarName {
// Sets the value for the variable
if len(actionResult.Result) > 0 {
//log.Printf("\n\n[DEBUG] SET EXEC VAR %s\n\n", execvar.Name)
workflowExecution.ExecutionVariables[index].Value = actionResult.Result
workflowExecution.Workflow.ExecutionVariables[index].Value = actionResult.Result
} else {
//log.Printf("\n\n[DEBUG] SKIPPING EXEC VAR\n\n")
}
break
}
}
}
log.Printf("[INFO][%s] Updating '%s' (%s) in workflow from %s to %s", workflowExecution.ExecutionId, actionResult.Action.Name, actionResult.Action.ID, workflowExecution.Results[outerindex].Status, actionResult.Status)
if workflowExecution.Results[outerindex].Status != actionResult.Status {
dbSave = true
}
actionResultBody, err := json.Marshal(actionResult)
if err == nil {
// Set cache for it too?
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID)
err = SetCache(ctx, cacheId, actionResultBody, 35)
if err != nil {
log.Printf("[ERROR] Failed setting cache for User Input to %s: %s", actionResult.Status, err)
} else {
//log.Printf("[DEBUG] Set cache for SUBFLOW action result %s", cacheId)
}
} else {
log.Printf("[ERROR] Failed marshaling action result for %s: %s", actionResult.Action.ID, err)
}
workflowExecution.Results[outerindex] = actionResult
} else {
workflowExecution.Results = append(workflowExecution.Results, actionResult)
}
} else {
log.Printf("[INFO][%s] Setting value of '%s' (INIT - %s) to %s (%d)", workflowExecution.ExecutionId, actionResult.Action.Label, actionResult.Action.ID, actionResult.Status, len(workflowExecution.Results))
workflowExecution.Results = append(workflowExecution.Results, actionResult)
}
// Auto fixing and ensuring the same isn't ran multiple times?
extraInputs := 0
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.Name == "User Input" && trigger.AppName == "User Input" {
extraInputs += 1
} else if trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow" {
extraInputs += 1
}
}
updateParentRan := false
if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extraInputs {
finished := true
lastResult := ""
// Doesn't have to be SUCCESS and FINISHED everywhere anymore.
//skippedNodes := false
for _, result := range workflowExecution.Results {
if result.Status == "EXECUTING" || result.Status == "WAITING" {
finished = false
break
}
if result.Status == "SUCCESS" {
lastResult = result.Result
}
}
if finished {
dbSave = true
if len(workflowExecution.ExecutionParent) == 0 {
//log.Printf("[INFO][%s] Execution in workflow %s finished (not subflow).", workflowExecution.ExecutionId, workflowExecution.Workflow.ID)
} else {
log.Printf("[INFO][%s] SubExecution of parentExecution %s in workflow %s finished (subflow).", workflowExecution.ExecutionId, workflowExecution.ExecutionParent, workflowExecution.Workflow.ID)
}
for actionIndex, action := range workflowExecution.Workflow.Actions {
for parameterIndex, param := range action.Parameters {
if param.Configuration {
//log.Printf("Cleaning up %s in %s", param.Name, action.Name)
workflowExecution.Workflow.Actions[actionIndex].Parameters[parameterIndex].Value = ""
}
}
}
workflowExecution.Result = lastResult
workflowExecution.Status = "FINISHED"
workflowExecution.CompletedAt = int64(time.Now().Unix())
if workflowExecution.LastNode == "" {
workflowExecution.LastNode = actionResult.Action.ID
}
// 1. Check if the LAST node is FAILURE or ABORTED or SKIPPED
// 2. If it's either of those, set the executionResult default value to DefaultReturnValue
valueToReturn := ""
if len(workflowExecution.Workflow.DefaultReturnValue) > 0 {
valueToReturn = workflowExecution.Workflow.DefaultReturnValue
for _, result := range workflowExecution.Results {
if result.Action.ID == workflowExecution.LastNode {
if result.Status == "ABORTED" || result.Status == "FAILURE" || result.Status == "SKIPPED" {
workflowExecution.Result = workflowExecution.Workflow.DefaultReturnValue
if len(workflowExecution.ExecutionParent) > 0 {
// 1. Find the parent workflow
// 2. Find the parent's existing value
log.Printf("[DEBUG] FOUND SUBFLOW WITH EXECUTIONPARENT %s!", workflowExecution.ExecutionParent)
}
} else {
valueToReturn = workflowExecution.Result
}
break
}
}
} else {
valueToReturn = workflowExecution.Result
}
// First: handle it in backend for loops
// 2nd: Handle it in worker for normal executions
/*
if len(workflowExecution.ExecutionParent) > 0 && (project.Environment == "onprem") {
//log.Printf("[DEBUG][%s] Got the result %s for subflow of %s. Check if this should be added to loop.", workflowExecution.ExecutionId, workflowExecution.Result, workflowExecution.ExecutionParent)
parentExecution, err := GetWorkflowExecution(ctx, workflowExecution.ExecutionParent)
if err == nil {
isLooping := false
for _, trigger := range parentExecution.Workflow.Triggers {
if trigger.ID == workflowExecution.ExecutionSourceNode {
for _, param := range trigger.Parameters {
if param.Name == "argument" && strings.Contains(param.Value, "$") && strings.Contains(param.Value, ".#") {
isLooping = true
break
}
}
break
}
}
if isLooping {
log.Printf("[DEBUG] Parentexecutions' subflow IS looping.")
}
}
} else
*/
if len(workflowExecution.ExecutionParent) > 0 && len(workflowExecution.ExecutionSourceAuth) > 0 && len(workflowExecution.ExecutionSourceNode) > 0 {
// Check if source node has "Wait for Results" set to true
log.Printf("[DEBUG][%s] Found execution parent %s for workflow '%s' (%s)", workflowExecution.ExecutionId, workflowExecution.ExecutionParent, workflowExecution.Workflow.Name, workflowExecution.Workflow.ID)
err = updateExecutionParent(ctx, workflowExecution.ExecutionParent, valueToReturn, workflowExecution.ExecutionSourceAuth, workflowExecution.ExecutionSourceNode, workflowExecution.ExecutionId)
if err != nil {
log.Printf("[ERROR][%s] Failed running update execution parent: %s", workflowExecution.ExecutionId, err)
} else {
updateParentRan = true
}
}
}
}
// Had to move this to run AFTER "updateExecutionParent()", as it's controlling whether a subflow should be updated or not
if actionResult.Status == "SUCCESS" && actionResult.Action.AppName == "shuffle-subflow" && !updateParentRan {
runCheck := false
for _, param := range actionResult.Action.Parameters {
if param.Name == "check_result" {
if param.Value == "true" {
runCheck = true
}
break
}
}
if runCheck {
var subflowData SubflowData
jsonerr := json.Unmarshal([]byte(actionResult.Result), &subflowData)
// Big blob to check cache & backend for more results
if jsonerr == nil && len(subflowData.Result) == 0 && !strings.Contains(actionResult.Result, "\"result\"") {
if project.Environment != "cloud" {
//Check cache for whether the execution actually finished or not
cacheKey := fmt.Sprintf("workflowexecution_%s", subflowData.ExecutionId)
value, err := GetCache(ctx, cacheKey)
if err == nil {
parsedValue := WorkflowExecution{}
cacheData := []byte(value.([]uint8))
err = json.Unmarshal(cacheData, &parsedValue)
if err == nil {
log.Printf("[INFO][%s] Found subflow result (1) %s for subflow %s in recheck from cache with %d results and result %s", workflowExecution.ExecutionId, parsedValue.Status, subflowData.ExecutionId, len(parsedValue.Results), parsedValue.Result)
if len(parsedValue.Result) > 0 {
subflowData.Result = parsedValue.Result
} else if parsedValue.Status == "FINISHED" {
subflowData.Result = "Subflow finished (PS: This is from worker autofill - happens if no actual result in subflow exec)"
}
}
// Check backend
//log.Printf("[INFO][%s] Found subflow result %s for subflow %s in recheck from cache with %d results and result %s", workflowExecution.ExecutionId, parsedValue.Status, subflowData.ExecutionId, len(parsedValue.Results), parsedValue.Result)
if len(subflowData.Result) == 0 && !strings.Contains(actionResult.Result, "\"result\"") {
log.Printf("[INFO][%s] No subflow result found in cache for subflow %s. Checking backend next", workflowExecution.ExecutionId, subflowData.ExecutionId)
if len(subflowData.ExecutionId) > 0 {
parsedValue, err := GetBackendexecution(ctx, subflowData.ExecutionId, subflowData.Authorization)
if err != nil {
log.Printf("[WARNING] Failed getting subflow execution from backend to verify: %s", err)
} else {
log.Printf("[INFO][%s] Found subflow result (2) %s for subflow %s in backend with %d results and result %s", workflowExecution.ExecutionId, parsedValue.Status, subflowData.ExecutionId, len(parsedValue.Results), parsedValue.Result)
if len(parsedValue.Result) > 0 {
subflowData.Result = parsedValue.Result
} else if parsedValue.Status == "FINISHED" {
subflowData.Result = "Subflow finished (PS: This is from worker autofill - happens if no actual result in subflow exec)"
}
}
}
}
}
}
}
log.Printf("[WARNING][%s] Sinkholing request of %s IF the subflow-result DOESNT have result.", workflowExecution.ExecutionId, actionResult.Action.Label)
// Just set the sinkholed data for some time in cache in case
// it will be necessary to use later. E.g. for wait for results
// + subflow data
newCacheKey := fmt.Sprintf("%s_%s_sinkholed_result", workflowExecution.ExecutionId, actionResult.Action.ID)
go SetCache(ctx, newCacheKey, []byte(actionResult.Result), 35)
if jsonerr == nil && len(subflowData.Result) == 0 && !strings.Contains(actionResult.Result, "\"result\"") {
log.Printf("[INFO][%s] NO RESULT FOR SUBFLOW RESULT - SETTING TO EXECUTING. Results: %d. Trying to find subexec in cache onprem", workflowExecution.ExecutionId, len(workflowExecution.Results))
// Finding the result, and removing it if it exists. "Sinkholing"
workflowExecution.Status = "EXECUTING"
newResults := []ActionResult{}
for _, result := range workflowExecution.Results {
if result.Action.ID == actionResult.Action.ID {
continue
}
newResults = append(newResults, result)
}
workflowExecution.Results = newResults
// Returning as we are waiting for the subflow to finish
return &workflowExecution, dbSave, nil
} else {
var subflowDataList []SubflowData
err = json.Unmarshal([]byte(actionResult.Result), &subflowDataList)
//if debug {
// log.Printf("\n\n\n\n\nSUBFLOW RESULT DATA: %#v\n\n\n\n\n", subflowData)
//}
// This is in case the list is not an actual list
if err != nil || len(subflowDataList) == 0 {
log.Printf("[WARNING][%s] NOT sinkholed from subflow result: %s", workflowExecution.ExecutionId, err)
for resultIndex, result := range workflowExecution.Results {
if result.Action.ID == actionResult.Action.ID {
workflowExecution.Results[resultIndex] = actionResult
break
}
}
} else {
log.Printf("[WARNING] LIST sinkholed (len: %d) for action %s (%s) - Should apply list setup for same as subflow without result! Set the execution back to EXECUTING and the action to WAITING, as it's already running. Waiting for each individual result to add to the list.", len(subflowDataList), actionResult.Action.Label, actionResult.Action.ID)
//log.Printf("\n\n\nRESULT: %#v\n\n\n", actionResult.Result)
// Set to executing, as the point is for the subflows themselves to update this part. This does NOT happen in the subflow, but in the parent workflow, which is waiting for results to be ingested, hence it's set to EXECUTING
// Setting to waiting, as it should be updated by child executions' fill-ins from their result when they finish
workflowExecution.Status = "EXECUTING"
amountFinished := 0
for _, subflowData := range subflowDataList {
if subflowData.ResultSet || len(subflowData.Result) > 0 {
amountFinished++
}
}
log.Printf("[DEBUG] %d / %d subflows finished with a result. If equal, status = SUCCESS", amountFinished, len(subflowDataList))
actionResultCache := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID)
if amountFinished >= len(subflowDataList) {
actionResult.Status = "SUCCESS"
// Force updating cache
parsedAction, err := json.Marshal(actionResult)
if err == nil {
SetCache(ctx, actionResultCache, parsedAction, 35)
}
dbSave = true
} else {
actionResult.Status = "WAITING"
DeleteCache(ctx, actionResultCache)
}
foundSubflow := false
for resultIndex, result := range workflowExecution.Results {
if result.Action.ID != actionResult.Action.ID {
continue
}
foundSubflow = true
workflowExecution.Results[resultIndex] = actionResult
actionResultBody, err := json.Marshal(actionResult)
if err == nil && actionResult.Status != "WAITING" {
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, actionResult.Action.ID)
err = SetCache(ctx, cacheId, actionResultBody, 35)
if err != nil {
log.Printf("[ERROR] Failed setting cache for SUBFLOW to WAITING: %s", err)
} else {
//log.Printf("[DEBUG] Set cache for SUBFLOW action result %s", cacheId)
}
} else {
//log.Printf("[ERROR] Failed marshalling action result for SUBFLOW to WAITING: %s", err)
}
break
}
if !foundSubflow {
log.Printf("[ERROR] Failed finding subflow in results for %s (%s). Setting it in cache so that it can be loaded.", actionResult.Action.Label, actionResult.Action.ID)
}
}
dbSave = true
}
}
}
workflowExecution, newDbSave := compressExecution(ctx, workflowExecution, "mid-cleanup")
if !dbSave {
dbSave = newDbSave
}
// Validates RERUN of single actions (new 2025)
// Identified by:
// 1. Predefined result from previous exec
// 2. Only ONE action
// 3. Every predefined result having result.Action.Category == "rerun"
if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 {
found := false
rerunFound := false
for _, result := range workflowExecution.Results {
if result.Action.Category == "rerun" {
rerunFound = true
}
// Find if the result for the single action exists or not
if result.Action.ID == workflowExecution.Workflow.Actions[0].ID {
found = true
}
}
if rerunFound && found {
// Continue -> this means finished check is ok
workflowExecution.Status = "FINISHED"
workflowExecution.CompletedAt = int64(time.Now().Unix())
dbSave = true
}
}
// Does it work to cache it here?
err = SetWorkflowExecution(ctx, workflowExecution, dbSave)
if err != nil {
log.Printf("[ERROR][%s] Failed saving execution to DB: %s", workflowExecution.ExecutionId, err)
}
// Should only apply a few seconds after execution, otherwise it's bascially spam.
if !skipExecutionCount && workflowExecution.Status == "FINISHED" {
//IncrementCache(ctx, workflowExecution.ExecutionOrg, "workflow_executions_finished")
}
// Should this be able to return errors?
//return &workflowExecution, dbSave, err
return &workflowExecution, dbSave, nil
}
func setExecutionVariable(actionResult ActionResult) bool {
if len(actionResult.Action.ExecutionVariable.Name) == 0 {
return false
}
if actionResult.Status != "SUCCESS" && actionResult.Status != "FINISHED" {
return false
}
setExecVar := true
if strings.Contains(actionResult.Result, "\"success\":") && !(strings.HasPrefix(actionResult.Result, "[{") && strings.HasSuffix(actionResult.Result, "}]")) {
type SubflowMapping struct {
Success bool `json:"success"`
}
var subflowData SubflowMapping
err := json.Unmarshal([]byte(actionResult.Result), &subflowData)
if err != nil {
log.Printf("[ERROR] Failed to map in set execvar name with success: %s", err)
setExecVar = false
} else {
if subflowData.Success == false {
setExecVar = false
}
}
}
if len(actionResult.Result) == 0 {
setExecVar = false
}
return setExecVar
}
// Finds execution results and parameters that are too large to manage and reduces them / saves data partly
func compressExecution(ctx context.Context, workflowExecution WorkflowExecution, saveLocationInfo string) (WorkflowExecution, bool) {
workerCompressExecution := os.Getenv("SHUFFLE_WORKER_COMPRESS")
if project.Environment == "worker" && (len(workerCompressExecution) == 0 || workerCompressExecution == "false"){
log.Printf("[DEBUG][%s] No need to make this execution any smaller", workflowExecution.ExecutionId)
return workflowExecution, false
}
//GetApp(ctx context.Context, id string, user User) (*WorkflowApp, error) {
//return workflowExecution, false
dbSave := false
tmpJson, err := json.Marshal(workflowExecution)
if err == nil {
if project.DbType != "opensearch" {
//log.Printf("[DEBUG] Result length is %d for execution Id %s, %s", len(tmpJson), workflowExecution.ExecutionId, saveLocationInfo)
if len(tmpJson) >= 1000000 {
// Clean up results' actions
//log.Printf("[DEBUG][%s](%s) ExecutionVariables size: %d, Result size: %d, executionArgument size: %d, Results size: %d", workflowExecution.ExecutionId, saveLocationInfo, len(workflowExecution.ExecutionVariables), len(workflowExecution.Result), len(workflowExecution.ExecutionArgument), len(workflowExecution.Results))
dbSave = true
//log.Printf("[WARNING][%s] Result length is too long (%d) when running %s! Need to reduce result size. Attempting auto-compression by saving data to disk.", workflowExecution.ExecutionId, len(tmpJson), saveLocationInfo)
actionId := "execution_argument"
//gs://shuffler.appspot.com/extra_specs/0373ed696a3a2cba0a2b6838068f2b80
//log.Printf("[WARNING] Couldn't find for %s. Should check filepath gs://%s/%s (size too big)", innerApp.ID, internalBucket, fullParsedPath)
// Result string `json:"result" datastore:"result,noindex"`
// Arbitrary reduction size
maxSize := 50000
bucketName := fmt.Sprintf("%s.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"))
//log.Printf("[DEBUG] Execution Argument length is %d for execution Id %s (%s)", len(workflowExecution.ExecutionArgument), workflowExecution.ExecutionId, saveLocationInfo)
if len(workflowExecution.ExecutionArgument) > maxSize {
itemSize := len(workflowExecution.ExecutionArgument)
baseResult := fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, actionId)
log.Printf("[DEBUG] len(executionArgument) is %d for execution Id %s", len(workflowExecution.ExecutionArgument), workflowExecution.ExecutionId)
fullParsedPath := fmt.Sprintf("large_executions/%s/%s_%s", workflowExecution.ExecutionOrg, workflowExecution.ExecutionId, actionId)
//log.Printf("[DEBUG] Saving value of %s to storage path %s", actionId, fullParsedPath)
bucket := project.StorageClient.Bucket(bucketName)
obj := bucket.Object(fullParsedPath)
w := obj.NewWriter(ctx)
if _, err := fmt.Fprint(w, workflowExecution.ExecutionArgument); err != nil {
log.Printf("[WARNING] Failed writing new exec file: %s", err)
workflowExecution.ExecutionArgument = baseResult
//continue
} else {
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Printf("[WARNING] Failed closing new exec file (2): %s", err)
workflowExecution.ExecutionArgument = baseResult
} else {
log.Printf("[DEBUG] Saved execution argument to %s", fullParsedPath)
workflowExecution.ExecutionArgument = fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "replace",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, actionId)
}
}
}
newResults := []ActionResult{}
//shuffle-large-executions
for _, item := range workflowExecution.Results {
//log.Printf("[DEBUG] Result length is %d for execution Id %s (%s)", len(item.Result), workflowExecution.ExecutionId, saveLocationInfo)
if len(item.Result) > maxSize {
//log.Printf("[WARNING][%s](%s) result length is larger than maxSize for %s (%d)", workflowExecution.ExecutionId, saveLocationInfo, item.Action.Label, len(item.Result))
itemSize := len(item.Result)
baseResult := fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, item.Action.ID)
// 1. Get the value and set it instead if it exists
// 2. If it doesn't exist, add it
_, err := getExecutionFileValue(ctx, workflowExecution, item)
if err == nil {
//log.Printf("[DEBUG][%s] Found execution file locally for '%s'. Not saving another.", workflowExecution.ExecutionId, item.Action.Label)
} else {
fullParsedPath := fmt.Sprintf("large_executions/%s/%s_%s", workflowExecution.ExecutionOrg, workflowExecution.ExecutionId, item.Action.ID)
//log.Printf("[DEBUG] (1) Saving value of %s to storage path %s", item.Action.ID, fullParsedPath)
bucket := project.StorageClient.Bucket(bucketName)
obj := bucket.Object(fullParsedPath)
w := obj.NewWriter(ctx)
//log.Printf("RES: ", item.Result)
if _, err := fmt.Fprint(w, item.Result); err != nil {
log.Printf("[WARNING][%s] Failed writing new exec file: %s", err, workflowExecution.ExecutionId)
item.Result = baseResult
newResults = append(newResults, item)
continue
}
// Close, just like writing a file.
if err := w.Close(); err != nil {
log.Printf("[WARNING][%s] Failed closing new exec file (1): %s", err, workflowExecution.ExecutionId)
item.Result = baseResult
newResults = append(newResults, item)
continue
}
}
item.Result = fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "replace",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, item.Action.ID)
// Setting an arbitrary decisionpoint to get it
// Backend will use this ID + action ID to get the data back
//item.Result = fmt.Sprintf("EXECUTION=%s", workflowExecution.ExecutionId)
}
newResults = append(newResults, item)
//log.Printf("[DEBUG][%s] newResults: %d and item labelled %s length is: %d", workflowExecution.ExecutionId, len(newResults), item.Action.Label, len(item.Result))
}
//log.Printf("[DEBUG][%s](%s) Overwriting executions results now! newResults length: %d", workflowExecution.ExecutionId, saveLocationInfo, len(newResults))
workflowExecution.Results = newResults
}
jsonString, err := json.Marshal(workflowExecution)
if err == nil {
//log.Printf("[DEBUG] Execution size: %d for %s", len(jsonString), workflowExecution.ExecutionId)
if len(jsonString) > 1000000 {
//log.Printf("[WARNING][%s] Execution size is still too large (%d) when running %s!", workflowExecution.ExecutionId, len(jsonString), saveLocationInfo)
//for _, action := range workflowExecution.Workflow.Actions {
// actionData, err := json.Marshal(action)
// if err == nil {
// //log.Printf("[DEBUG] Action Size for %s (%s - %s): %d", action.Label, action.Name, action.ID, len(actionData))
// }
//}
for resultIndex, result := range workflowExecution.Results {
//resultData, err := json.Marshal(result)
//_ = resultData
actionData, err := json.Marshal(result.Action)
if err == nil {
// log.Printf("[DEBUG] Result Size (%s - action: %d): %d. Value size: %d", result.Action.Label, len(resultData), len(actionData), len(result.Result))
}
if len(actionData) > 10000 {
for paramIndex, param := range result.Action.Parameters {
if len(param.Value) > 10000 {
workflowExecution.Results[resultIndex].Action.Parameters[paramIndex].Value = "Size too large. Removed."
}
}
}
}
}
}
} else {
// OpenSearch (on-premise) handling
// Offload execution_argument to file independently of total size.
// Lucene rejects terms > 32766 bytes, so any execution_argument
// larger than 32500 bytes will cause an illegal_argument_exception.
if len(workflowExecution.ExecutionArgument) > 32500 && !strings.Contains(workflowExecution.ExecutionArgument, "Result too large to handle") {
dbSave = true
itemSize := len(workflowExecution.ExecutionArgument)
actionId := "execution_argument"
basepath := os.Getenv("SHUFFLE_FILE_LOCATION")
if len(basepath) == 0 {
basepath = "files"
}
fullParsedPath := fmt.Sprintf("large_executions/%s/%s_%s", workflowExecution.ExecutionOrg, workflowExecution.ExecutionId, actionId)
localPath := fmt.Sprintf("%s/%s", basepath, fullParsedPath)
log.Printf("[DEBUG][%s] Offloading execution_argument (%d bytes) to file %s", workflowExecution.ExecutionId, itemSize, localPath)
replacementJson := fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "replace",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, actionId)
if err := ioutil.WriteFile(localPath, []byte(workflowExecution.ExecutionArgument), 0644); err != nil {
dirPath := fmt.Sprintf("%s/large_executions/%s", basepath, workflowExecution.ExecutionOrg)
if mkdirErr := os.MkdirAll(dirPath, 0755); mkdirErr != nil {
log.Printf("[WARNING] Failed creating directory %s: %s (original write error: %s)", dirPath, mkdirErr, err)
} else if retryErr := ioutil.WriteFile(localPath, []byte(workflowExecution.ExecutionArgument), 0644); retryErr != nil {
log.Printf("[WARNING] Failed writing execution_argument file after creating directory: %s", retryErr)
} else {
workflowExecution.ExecutionArgument = replacementJson
}
} else {
workflowExecution.ExecutionArgument = replacementJson
}
}
// Offload individual Results[].Result to file independently of total size.
// Lucene rejects terms > 32766 bytes, so any action result
// larger than 32500 bytes will cause an illegal_argument_exception.
basepath := os.Getenv("SHUFFLE_FILE_LOCATION")
if len(basepath) == 0 {
basepath = "files"
}
newResults := []ActionResult{}
for _, item := range workflowExecution.Results {
if len(item.Result) > 32500 && !strings.Contains(item.Result, "Result too large to handle") {
dbSave = true
itemSize := len(item.Result)
fullParsedPath := fmt.Sprintf("large_executions/%s/%s_%s", workflowExecution.ExecutionOrg, workflowExecution.ExecutionId, item.Action.ID)
localPath := fmt.Sprintf("%s/%s", basepath, fullParsedPath)
log.Printf("[DEBUG][%s] Offloading result for action %s (%d bytes) to file %s", workflowExecution.ExecutionId, item.Action.Label, itemSize, localPath)
replacementJson := fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "replace",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, item.Action.ID)
if err := ioutil.WriteFile(localPath, []byte(item.Result), 0644); err != nil {
dirPath := fmt.Sprintf("%s/large_executions/%s", basepath, workflowExecution.ExecutionOrg)
if mkdirErr := os.MkdirAll(dirPath, 0755); mkdirErr != nil {
log.Printf("[WARNING][%s] Failed creating directory %s: %s (original write error: %s)", workflowExecution.ExecutionId, dirPath, mkdirErr, err)
} else if retryErr := ioutil.WriteFile(localPath, []byte(item.Result), 0644); retryErr != nil {
log.Printf("[WARNING][%s] Failed writing result file after creating directory: %s", workflowExecution.ExecutionId, retryErr)
} else {
item.Result = replacementJson
}
} else {
item.Result = replacementJson
}
}
newResults = append(newResults, item)
}
workflowExecution.Results = newResults
// Offload workflowExecution.Result to file independently of total size.
// Lucene rejects terms > 32766 bytes, so the Result field
// larger than 32500 bytes will cause an illegal_argument_exception.
if len(workflowExecution.Result) > 32500 && !strings.Contains(workflowExecution.Result, "Result too large to handle") {
dbSave = true
itemSize := len(workflowExecution.Result)
actionId := "execution_result"
fullParsedPath := fmt.Sprintf("large_executions/%s/%s_%s", workflowExecution.ExecutionOrg, workflowExecution.ExecutionId, actionId)
localPath := fmt.Sprintf("%s/%s", basepath, fullParsedPath)
log.Printf("[DEBUG][%s] Offloading Result field (%d bytes) to file %s", workflowExecution.ExecutionId, itemSize, localPath)
replacementJson := fmt.Sprintf(`{
"success": false,
"reason": "Result too large to handle (https://github.com/frikky/shuffle/issues/171).",
"size": %d,
"extra": "replace",
"id": "%s_%s"
}`, itemSize, workflowExecution.ExecutionId, actionId)
if err := ioutil.WriteFile(localPath, []byte(workflowExecution.Result), 0644); err != nil {
dirPath := fmt.Sprintf("%s/large_executions/%s", basepath, workflowExecution.ExecutionOrg)
if mkdirErr := os.MkdirAll(dirPath, 0755); mkdirErr != nil {
log.Printf("[WARNING] Failed creating directory %s: %s (original write error: %s)", dirPath, mkdirErr, err)
} else if retryErr := ioutil.WriteFile(localPath, []byte(workflowExecution.Result), 0644); retryErr != nil {
log.Printf("[WARNING] Failed writing Result file after creating directory: %s", retryErr)
} else {
workflowExecution.Result = replacementJson
}
} else {
workflowExecution.Result = replacementJson
}
}
// Trim action parameter values > 32500 bytes (Lucene keyword term limit).
// Parameters are keyword fields and hit the same 32766-byte Lucene limit.
for resultIndex, result := range workflowExecution.Results {
for paramIndex, param := range result.Action.Parameters {
if len(param.Value) > 32500 {
log.Printf("[DEBUG][%s] Trimming parameter %s in action %s (size: %d bytes)", workflowExecution.ExecutionId, param.Name, result.Action.Label, len(param.Value))
workflowExecution.Results[resultIndex].Action.Parameters[paramIndex].Value = "Size too large. Removed."
}
}
}
jsonString, err := json.Marshal(workflowExecution)
if err == nil {
if debug {
log.Printf("[DEBUG] Execution size: %d for %s", len(jsonString), workflowExecution.ExecutionId)
}
}
}
}
return workflowExecution, dbSave
}
// Recursively finds the child nodes of a node in execution and returns their ID.
// Used if e.g. a node in a branch is exited, and all children have to be stopped
// Also used during startup of a workflow to set all nodes to be SKIPPED that aren't in immediate use
func FindChildNodes(workflow Workflow, nodeId string, parents, handledBranches []string) []string {
allChildren := []string{nodeId}
// 1. Find children of this specific node
// 2. Find the children of those nodes etc.
// 3. Sort it in the right order to handle merges properly
for _, branch := range workflow.Branches {
if branch.SourceID == nodeId {
if ArrayContains(parents, branch.DestinationID) {
continue
}
parents = append(parents, branch.SourceID)
if ArrayContains(handledBranches, branch.ID) {
continue
}
allChildren = append(allChildren, branch.DestinationID)
handledBranches = append(handledBranches, branch.ID)
childNodes := FindChildNodes(workflow, branch.DestinationID, parents, handledBranches)
for _, bottomChild := range childNodes {
found := false
for _, topChild := range allChildren {
if topChild == bottomChild {
found = true
break
}
}
if !found {
allChildren = append(allChildren, bottomChild)
}
}
}
}
// Remove potential duplicates
newNodes := []string{}
for _, tmpnode := range allChildren {
if tmpnode == nodeId {
continue
}
found := false
for _, newnode := range newNodes {
if newnode == tmpnode {
found = true
break
}
}
if !found {
newNodes = append(newNodes, tmpnode)
}
}
return newNodes
}
func GetExecutionbody(body []byte) string {
parsedBody := string(body)
// Specific weird newline issues
if strings.Contains(parsedBody, "choice") {
if strings.Count(parsedBody, `\\n`) > 2 {
parsedBody = strings.Replace(parsedBody, `\\n`, "", -1)
}
if strings.Count(parsedBody, `\u0022`) > 2 {
parsedBody = strings.Replace(parsedBody, `\u0022`, `"`, -1)
}
if strings.Count(parsedBody, `\\"`) > 2 {
parsedBody = strings.Replace(parsedBody, `\\"`, `"`, -1)
}
if strings.Contains(parsedBody, `"extra": "{`) {
parsedBody = strings.Replace(parsedBody, `"extra": "{`, `"extra": {`, 1)
parsedBody = strings.Replace(parsedBody, `}"}`, `}}`, 1)
}
}
// Replaces dots in string when it's key specifically has a dot
// FIXME: Do this with key recursion and key replacements only
pattern := regexp.MustCompile(`\"(\w+)\.(\w+)\":`)
found := pattern.FindAllString(parsedBody, -1)
for _, item := range found {
newItem := strings.Replace(item, ".", "_", -1)
parsedBody = strings.Replace(parsedBody, item, newItem, -1)
}
if !strings.HasPrefix(parsedBody, "{") && !strings.HasPrefix(parsedBody, "[") && strings.Contains(parsedBody, "=") {
//log.Printf("[DEBUG] Trying to make string %s to json (skipping if XML, doing queries & k:v)", parsedBody)
// Dumb XML handler
if strings.HasPrefix(strings.TrimSpace(parsedBody), "<") && strings.HasSuffix(strings.TrimSpace(parsedBody), ">") {
log.Printf("[DEBUG] XML detected. Not parsing anyything.")
return parsedBody
}
newbody := map[string]string{}
for _, item := range strings.Split(parsedBody, "&") {
//log.Printf("Handling item: %s", item)
if !strings.Contains(item, "=") {
newbody[item] = ""
continue
}
bodySplit := strings.Split(item, "=")
if len(bodySplit) == 2 {
newbody[bodySplit[0]] = bodySplit[1]
} else {
newbody[item] = ""
}
}
jsonString, err := json.Marshal(newbody)
if err != nil {
log.Printf("[ERROR] Failed marshaling queries: %s: %s", newbody, err)
} else {
parsedBody = string(jsonString)
}
//fmt.Printf(err)
//log.Printf("BODY: %s", newbody)
}
// Check bad characters in keys
// FIXME: Re-enable this when it's safe.
//log.Printf("Input: %s", parsedBody)
parsedBody = string(FixBadJsonBody([]byte(parsedBody)))
//log.Printf("Output: %s", parsedBody)
return parsedBody
}
// Can't just regex out stuff due to unicode problems with other languages
func handleKeyRemoval(key string) string {
abolish := []string{"!", "@", "#", "$", "%", "~", "|", "^", "&", "*", "(", ")", "[", "]", "{", "}", "<", ">", "+", "=", "?", ".", ",", "/", "\\", "'"}
for _, remove := range abolish {
key = strings.Replace(key, remove, "", -1)
}
return key
}
// https://www.codemio.com/2021/02/advanced-golang-tutorials-dynamic-json-parsing.html
func handleJSONObject(object interface{}, key, totalObject string) string {
currentObject := ""
key = handleKeyRemoval(key)
switch t := object.(type) {
case int:
currentObject += fmt.Sprintf(`"%s": %d, `, key, t)
if len(key) == 0 {
currentObject += fmt.Sprintf(`%d, `, t)
}
case int64:
currentObject += fmt.Sprintf(`"%s": %d, `, key, t)
if len(key) == 0 {
currentObject += fmt.Sprintf(`%d, `, t)
}
case float64:
tmpObject := fmt.Sprintf(`"%s": %f, `, key, t)
if len(key) == 0 {
tmpObject = fmt.Sprintf(`%f, `, t)
}
if strings.HasSuffix(tmpObject, "000000, ") {
tmpObject = tmpObject[0 : len(tmpObject)-9]
tmpObject += ", "
}
currentObject += tmpObject
case bool:
if len(key) == 0 {
currentObject += fmt.Sprintf(`%v, `, t)
} else {
currentObject += fmt.Sprintf(`"%s": %v, `, key, t)
}
case string:
if len(key) == 0 {
currentObject += fmt.Sprintf(`"%s", `, t)
} else {
currentObject += fmt.Sprintf(`"%s": "%s", `, key, t)
}
case map[string]interface{}:
if len(key) == 0 {
currentObject += fmt.Sprintf(`{`)
} else {
currentObject += fmt.Sprintf(`"%s": {`, key)
}
for k, v := range t {
currentObject = handleJSONObject(v, k, currentObject)
}
if len(currentObject) > 3 {
currentObject = currentObject[0 : len(currentObject)-2]
}
currentObject += "}, "
case []interface{}:
if len(key) == 0 {
currentObject += fmt.Sprintf(`[`)
} else {
currentObject += fmt.Sprintf(`"%s": [`, key)
}
for _, v := range t {
currentObject = handleJSONObject(v, "", currentObject)
}
if len(currentObject) > 3 {
currentObject = currentObject[0 : len(currentObject)-2]
}
currentObject += "], "
default:
log.Printf("[ERROR] Missing handler for type %s in app framework - key: %s", t, key)
}
totalObject += currentObject
return totalObject
}
func FixBadJsonBody(parsedBody []byte) []byte {
if os.Getenv("SHUFFLE_JSON_PARSER") != "parse" {
return parsedBody
}
// NOT handling data that starts as a loop for now: [] instead of {} as outer wrapper.
// Lists and all other types do work inside the JSON, and are rebuilt with a new key (if applicable).
if !strings.HasPrefix(string(parsedBody), "{") {
return parsedBody
}
var results map[string]interface{}
err := json.Unmarshal([]byte(parsedBody), &results)
if err != nil {
log.Printf("[WARNING] Failed parsing data: %s", err)
return parsedBody
}
totalObject := "{"
for key, value := range results {
_ = value
totalObject = handleJSONObject(value, key, totalObject)
}
if len(totalObject) > 3 {
totalObject = totalObject[0 : len(totalObject)-2]
}
totalObject += "}"
//log.Printf("Auto sanitized keys.: %s", totalObject)
//for _, result := range results {
// // But if you don't know the field types, you can use type switching to determine (safe):
// // Keep in mind that, since this is a map, the order is not guaranteed.
// fmt.Printf("\nType Switching: ")
// for k := range result {
// }
// fmt.Printf("------------------------------")
//}
return []byte(totalObject)
}
func ValidateSwagger(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in validate swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to validate swagger (shared): %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type versionCheck struct {
Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"`
SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"`
OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"`
}
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version versionCheck
re := regexp.MustCompile("[[:^ascii:]]")
//re := regexp.MustCompile("[[:^unicode:]]")
t := re.ReplaceAllLiteralString(string(body), "")
log.Printf("[DEBUG] App build API length: %d. Cleanup length: %d", len(string(body)), len(t))
body = []byte(t)
isJson := false
err = json.Unmarshal(body, &version)
if err != nil {
log.Printf("[WARNING] Json API upload err: %s", err)
body = []byte(strings.Replace(string(body), "\\/", "/", -1))
err = yaml.Unmarshal(body, &version)
if err != nil {
log.Printf("[WARNING] Yaml error (3): %s", err)
//if len(string(body)) < 500 {
// log.Printf("%s",
//}
parsedResult := ResultChecker{
Success: false,
Reason: fmt.Sprintf("Issue in JSON/YAML: %s", err),
}
resp.WriteHeader(422)
marshalledResult, err := json.Marshal(parsedResult)
if err == nil {
resp.Write(marshalledResult)
} else {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi to json and yaml. Is version defined?: %s"}`, err)))
}
return
} else {
log.Printf("[INFO] Successfully parsed YAML (3)!")
}
} else {
isJson = true
log.Printf("[INFO] Successfully parsed JSON!")
}
if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 {
version.Swagger = version.SwaggerVersion
}
log.Printf("[INFO] Version: %s", version)
log.Printf("[INFO] OpenAPI: %s", version.OpenAPI)
if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") {
log.Printf("[INFO] Handling v3 API")
swaggerLoader := openapi3.NewSwaggerLoader()
swaggerLoader.IsExternalRefsAllowed = true
//swagger, err := swaggerLoader.LoadSwaggerFromData(body)
swagger := &openapi3.Swagger{}
swagger, err = swaggerLoader.LoadSwaggerFromData(body)
if err != nil {
log.Printf("[WARNING] Failed to convert v3 API: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
hasher := md5.New()
hasher.Write(body)
idstring := hex.EncodeToString(hasher.Sum(nil))
log.Printf("[INFO] Swagger v3 validation success with ID %s and %d paths!", idstring, len(swagger.Paths))
if !isJson {
log.Printf("[INFO] NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring)
}
swaggerdata, err := json.Marshal(swagger)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling v3 data: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling swaggerv3 data: %s"}`, err)))
return
}
parsed := ParsedOpenApi{
ID: idstring,
Body: string(swaggerdata),
}
ctx := GetContext(request)
err = SetOpenApiDatastore(ctx, idstring, parsed)
if err != nil {
log.Printf("[WARNING] Failed uploading openapi to datastore: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err)))
return
}
log.Printf("[INFO] Successfully set OpenAPI with name %s and ID %s", swagger.Info.Title, idstring)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, idstring)))
return
} else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") {
// Convert
log.Printf("[WARNING] Handling v2 API")
swagger := openapi2.Swagger{}
//log.Printf(string(body))
err = json.Unmarshal(body, &swagger)
if err != nil {
log.Printf("[WARNING] Json error for v2 - trying yaml next: %s", err)
err = yaml.Unmarshal([]byte(body), &swagger)
if err != nil {
log.Printf("[WARNING] Yaml error (4): %s", err)
if strings.Contains(fmt.Sprintf("%s", err), "cannot unmarshal") {
log.Printf("[WARNING] Failed unmarshaling v2 data: %s - this is allowed.", err)
} else {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err)))
return
}
} else {
log.Printf("Found valid yaml!")
}
}
swaggerv3, err := openapi2conv.ToV3Swagger(&swagger)
if err != nil {
log.Printf("[WARNING] Failed converting from openapi2 to 3: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed converting from openapi2 to openapi3: %s"}`, err)))
return
}
swaggerdata, err := json.Marshal(swaggerv3)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling v3 from v2 data: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling swaggerv3 data: %s"}`, err)))
return
}
hasher := md5.New()
hasher.Write(swaggerdata)
idstring := hex.EncodeToString(hasher.Sum(nil))
if !isJson {
log.Printf("[WARNING] FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring)
}
log.Printf("[INFO] Swagger v2 -> v3 validation success with ID %s!", idstring)
parsed := ParsedOpenApi{
ID: idstring,
Body: string(swaggerdata),
}
ctx := GetContext(request)
err = SetOpenApiDatastore(ctx, idstring, parsed)
if err != nil {
log.Printf("[WARNING] Failed uploading openapi2 to datastore: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, idstring)))
return
}
/*
else {
log.Printf("Swagger / OpenAPI version %s is not supported or there is an error.", version.Swagger)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Swagger version %s is not currently supported"}`, version.Swagger)))
return
}
*/
// save the openapi ID
resp.WriteHeader(422)
resp.Write([]byte(`{"success": false}`))
}
// Recursively finds child nodes inside sub workflows
func GetReplacementNodes(ctx context.Context, execution WorkflowExecution, trigger Trigger, lastTriggerName string) ([]Action, []Branch, string) {
if execution.ExecutionOrg == "" {
execution.ExecutionOrg = execution.Workflow.OrgId
}
selectedWorkflow := ""
workflowAction := ""
for _, param := range trigger.Parameters {
if param.Name == "workflow" {
selectedWorkflow = param.Value
}
if param.Name == "startnode" {
workflowAction = param.Value
}
}
if len(selectedWorkflow) == 0 {
return []Action{}, []Branch{}, ""
}
// Authenticating and such
workflow, err := GetWorkflow(ctx, selectedWorkflow)
if err != nil {
return []Action{}, []Branch{}, ""
}
orgFound := false
if workflow.ExecutingOrg.Id == execution.ExecutionOrg {
orgFound = true
} else if workflow.OrgId == execution.ExecutionOrg {
orgFound = true
} else {
for _, org := range workflow.Org {
if org.Id == execution.ExecutionOrg {
orgFound = true
break
}
}
}
if !orgFound {
log.Printf("[WARNING] Auth for subflow is bad. %s (orig) vs %s", execution.ExecutionOrg, workflow.OrgId)
return []Action{}, []Branch{}, ""
}
//childNodes = FindChildNodes(workflowExecution, actionResult.Action.ID)
//log.Printf("FIND CHILDNODES OF STARTNODE %s", workflowAction)
workflowExecution := WorkflowExecution{
Workflow: *workflow,
}
childNodes := FindChildNodes(workflowExecution.Workflow, workflowAction, []string{}, []string{})
newActions := []Action{}
branches := []Branch{}
// FIXME: Bad lastnode check. Need to go to the bottom of workflows and check max steps away from parent
lastNode := ""
for _, nodeId := range childNodes {
for _, action := range workflow.Actions {
if nodeId == action.ID {
newActions = append(newActions, action)
break
}
}
for _, branch := range workflow.Branches {
if branch.SourceID == nodeId {
branches = append(branches, branch)
}
}
lastNode = nodeId
}
found := false
for actionIndex, action := range newActions {
if lastNode == action.ID {
//actions[actionIndex].Name = trigger.Name
newActions[actionIndex].Label = lastTriggerName
//trigger.Label
found = true
}
}
if !found {
log.Printf("SHOULD CHECK TRIGGERS FOR LASTNODE!")
}
log.Printf("[INFO] Found %d actions and %d branches in subflow", len(newActions), len(branches))
if len(newActions) == len(childNodes) {
return newActions, branches, lastNode
} else {
// Adding information about triggers if subflow
changed := false
for _, nodeId := range childNodes {
for triggerIndex, trigger := range workflow.Triggers {
if trigger.AppName == "Shuffle Workflow" {
if nodeId == trigger.ID {
replaceActions := false
workflowAction := ""
for _, param := range trigger.Parameters {
if param.Name == "argument" && !strings.Contains(param.Value, ".#") {
replaceActions = true
}
if param.Name == "startnode" {
workflowAction = param.Value
}
}
if replaceActions {
replacementNodes, newBranches, lastNode := GetReplacementNodes(ctx, workflowExecution, trigger, lastTriggerName)
if len(replacementNodes) > 0 {
//workflowExecution.Workflow.Actions = append(workflowExecution.Workflow.Actions, action)
//lastnode = replacementNodes[0]
// Have to validate in case it's the same workflow and such
for _, action := range replacementNodes {
found := false
for subActionIndex, subaction := range newActions {
if subaction.ID == action.ID {
found = true
//newActions[subActionIndex].Name = action.Name
newActions[subActionIndex].Label = action.Label
break
}
}
if !found {
action.SubAction = true
newActions = append(newActions, action)
}
}
for _, branch := range newBranches {
workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, branch)
}
// Append branches:
// parent -> new inner node (FIRST one)
for branchIndex, branch := range workflowExecution.Workflow.Branches {
if branch.DestinationID == trigger.ID {
log.Printf("REPLACE DESTINATION WITH %s!!", workflowAction)
workflowExecution.Workflow.Branches[branchIndex].DestinationID = workflowAction
branches = append(branches, workflowExecution.Workflow.Branches[branchIndex])
}
if branch.SourceID == trigger.ID {
log.Printf("REPLACE SOURCE WITH LASTNODE %s!!", lastNode)
workflowExecution.Workflow.Branches[branchIndex].SourceID = lastNode
branches = append(branches, workflowExecution.Workflow.Branches[branchIndex])
}
}
// Remove the trigger
workflowExecution.Workflow.Triggers = append(workflowExecution.Workflow.Triggers[:triggerIndex], workflowExecution.Workflow.Triggers[triggerIndex+1:]...)
workflow.Triggers = append(workflow.Triggers[:triggerIndex], workflow.Triggers[triggerIndex+1:]...)
changed = true
}
}
}
}
}
}
if changed {
return newActions, branches, lastNode
}
}
return []Action{}, []Branch{}, ""
}
// Uses a simple way to be able to modify the encryption key being used
// FIXME: Investigate better ways of handling EVERYTHING related to encryption
// E.g. rolling keys and such
func create32Hash(key string) ([]byte, error) {
encryptionModifier := os.Getenv("SHUFFLE_ENCRYPTION_MODIFIER")
if len(encryptionModifier) == 0 {
return []byte{}, errors.New(fmt.Sprintf("No encryption modifier set. Define env SHUFFLE_ENCRYPTION_MODIFIER to some random string and NEVER change it to start using encrypted auth."))
}
key += encryptionModifier
hasher := md5.New()
hasher.Write([]byte(key))
return []byte(hex.EncodeToString(hasher.Sum(nil))), nil
}
func HandleKeyEncryption(data []byte, passphrase string) ([]byte, error) {
key, err := create32Hash(passphrase)
if err != nil {
log.Printf("[WARNING] Skipped hashing in encrypt: %s", err)
return []byte{}, err
}
block, err := aes.NewCipher(key)
if err != nil {
log.Printf("[WARNING] Error generating ciphertext: %s", err)
return []byte{}, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
log.Printf("[WARNING] Error creating new GCM from block: %s", err)
return []byte{}, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
log.Printf("[WARNING] Error reading GCM nonce: %s", err)
return []byte{}, err
}
ciphertext := gcm.Seal(nonce, nonce, data, nil)
// base64 encoding to ensure we can store it as a string
parsedValue := base64.StdEncoding.EncodeToString(ciphertext)
return []byte(parsedValue), nil
}
func HandleKeyDecryption(data []byte, passphrase string) ([]byte, error) {
//if debug {
// log.Printf("[DEBUG] Passphrase: %s", passphrase)
// log.Printf("Decrypting key: %s", data)
//}
key, err := create32Hash(passphrase)
if err != nil {
log.Printf("[ERROR] Failed hashing in decrypt: %s", err)
return []byte{}, err
}
block, err := aes.NewCipher(key)
if err != nil {
log.Printf("[ERROR] Error creating cipher from key in decryption: %s", err)
return []byte{}, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
log.Printf("[ERROR] Error creating new GCM block in decryption: %s", err)
return []byte{}, err
}
parsedData, err := base64.StdEncoding.DecodeString(string(data))
if err != nil {
//log.Printf("[WARNING] Failed base64 decode for auth key '%s': '%s'. Data: '%s'. Returning as if this is valid.", data, err, string(data))
//return []byte{}, err
return data, nil
}
nonceSize := gcm.NonceSize()
if nonceSize > len(parsedData) {
//log.Printf("[ERROR] Nonce size is larger than parsed data in decryption. Returning as if this is valid. This should _never_ happen, but typically only happens IF the source data is invalid (e.g. 1/20 keys)")
//if debug {
// log.Printf("Returned: '%s'. Len %d vs %d", string(parsedData), nonceSize, len(parsedData))
//}
return data, nil
}
nonce, ciphertext := parsedData[:nonceSize], parsedData[nonceSize:]
plaintext, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
//log.Printf("[ERROR] Error reading decryptionkey: %s - nonce: %s, ciphertext: %s", err, nonce, ciphertext)
//log.Printf("[ERROR] Error reading decryptionkey: %s - nonce: %s", err, nonce)
return []byte{}, err
}
return plaintext, nil
}
func HandleListCacheKeys(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, usererr := HandleApiAuthentication(resp, request)
if usererr != nil {
log.Printf("[AUDIT] Api authentication failed in list datastore keys: %s. Allowing continue in case category is public", usererr)
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
//return
} else {
if user.Role != "admin" && !user.SupportAccess {
log.Printf("[AUDIT] User %s (%s) tried to list cache keys without admin role", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Admin required"}`))
return
}
}
//for key, value := range data.Apps {
var orgId string
category := ""
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
} else {
if location[4] == "category" && len(location) > 5 {
category = location[5]
if strings.Contains(category, "?") {
category = strings.Split(category, "?")[0]
}
} else {
orgId = location[4]
}
}
}
// Overwriting, as we don't want it to work that way
// Should use Org-Id header instead
orgId = user.ActiveOrg.Id
categoryList, categoryOk := request.URL.Query()["category"]
if categoryOk && len(categoryList) > 0 {
//category = categoryList[0]
category = categoryList[0]
}
orgQuery, orgOk := request.URL.Query()["org_id"]
if orgOk && len(orgQuery) > 0 {
orgId = orgQuery[0]
}
if usererr != nil {
if len(category) == 0 || category == "default" {
log.Printf("[WARNING] No category provided in request. Returning 400.")
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No category provided"}`))
return
}
// NEED to check the org etc
if len(orgId) == 0 {
log.Printf("[WARNING] No org ID provided in request. Returning 400.")
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No org ID provided"}`))
return
}
}
// Requires being admin
if strings.ToLower(category) == "protected" {
if user.Role != "admin" {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Admin required to access protected category"}`))
return
}
}
ctx := GetContext(request)
org, err := GetOrg(ctx, orgId)
if err != nil {
log.Printf("[INFO] Organization '%s' doesn't exist: %s", orgId, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
maxAmount := 100
top, topOk := request.URL.Query()["top"]
if topOk && len(top) > 0 {
val, err := strconv.Atoi(top[0])
if err == nil {
maxAmount = val
}
}
cursor := ""
cursorList, cursorOk := request.URL.Query()["cursor"]
if cursorOk && len(cursorList) > 0 {
cursor = cursorList[0]
}
keys := []CacheKeyData{}
newCursor := ""
isSuccess := true
if keyList, keyOk := request.URL.Query()["key"]; keyOk && len(keyList) > 0 {
key := keyList[0]
cacheId := fmt.Sprintf("%s_%s", user.ActiveOrg.Id, key)
if len(category) > 0 {
cacheId = fmt.Sprintf("%s_%s_%s", user.ActiveOrg.Id, key, category)
}
cacheItem, err := GetDatastoreKey(ctx, cacheId, category)
if err != nil {
isSuccess = false
}
keys = []CacheKeyData{
*cacheItem,
}
} else {
if debug {
log.Printf("[DEBUG] Looking for keys in org %s and category %s", org.Id, category)
}
keys, newCursor, err = GetAllCacheKeys(ctx, org.Id, category, maxAmount, cursor)
if err != nil {
isSuccess = false
}
}
// This is NOT required unless automation/other config is set.
foundCategories := []string{}
categoryConfig := &DatastoreCategoryUpdate{}
if len(category) > 0 && category != "default" {
foundCategories = append(foundCategories, category)
categoryConfig, err = GetDatastoreCategoryConfig(ctx, org.Id, category)
if err != nil {
//if debug {
// log.Printf("[WARNING] Failed to get category config for org %s: %s", org.Id, err)
//}
}
} else {
allCategories, err := GetDatastoreCategories(ctx, org.Id)
if err == nil {
for _, cat := range allCategories {
if len(cat.Category) <= 1 || cat.Category == "default" {
continue
}
foundCategories = append(foundCategories, cat.Category)
}
}
for _, key := range keys {
if len(key.Category) <= 1 || key.Category == "default" {
continue
}
if ArrayContains(foundCategories, key.Category) {
continue
}
foundCategories = append(foundCategories, key.Category)
}
}
if orgId != user.ActiveOrg.Id {
if !categoryConfig.Settings.Public {
sourceExecution, sourceExecutionOk := request.URL.Query()["execution_id"]
sourceAuth, sourceAuthOk := request.URL.Query()["authorization"]
if !sourceAuthOk || !sourceExecutionOk {
log.Printf("[AUDIT] User %s (%s) tried to list cache keys for org %s without access", user.Username, user.Id, orgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "This category is no longer public."}`))
return
}
foundExec, err := GetWorkflowExecution(ctx, sourceExecution[0])
if err != nil {
log.Printf("[WARNING] Failed getting exec during cache set: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "No permission to get execution (2)"}`))
return
}
if sourceAuth[0] != foundExec.Authorization {
log.Printf("[INFO] Execution auth %s and %s don't match", foundExec.Authorization, sourceAuth[0])
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication (3)"}`))
return
}
if len(foundExec.ExecutionOrg) == 0 {
log.Printf("[WARNING] Execution %s doesn't have an org set", foundExec.ExecutionId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication (4)"}`))
return
}
}
// Cleanup just in case
categoryConfig = &DatastoreCategoryUpdate{}
for keyIndex, _ := range keys {
keys[keyIndex].WorkflowId = ""
keys[keyIndex].ExecutionId = ""
keys[keyIndex].PublicAuthorization = ""
keys[keyIndex].SuborgDistribution = []string{}
}
}
// Sort categories
sort.SliceStable(foundCategories, func(i, j int) bool {
return foundCategories[i] < foundCategories[j]
})
newReturn := CacheReturn{
Success: isSuccess,
Keys: keys,
Cursor: newCursor,
Amount: len(keys),
TotalAmount: -1,
Category: category,
Config: *categoryConfig,
Categories: foundCategories,
}
outputTypeList, outputTypeOk := request.URL.Query()["type"]
if outputTypeOk && len(outputTypeList) > 0 {
outputType := outputTypeList[0]
if outputType == "ndjson" || outputType == "csv" || outputType == "raw" {
outputString := ""
for _, key := range newReturn.Keys {
if len(key.Value) == 0 {
continue
}
newValue := strings.ReplaceAll(strings.ReplaceAll(key.Value, "\\n", "\n"), "\\r", "\r")
newValue = strings.ReplaceAll(strings.ReplaceAll(newValue, "\n", "\\n"), "\r", "\\r")
outputString += newValue + "\n"
}
// This forces browsers to download for some reason?
//resp.Header().Set("Content-Type", "application/x-ndjson")
resp.WriteHeader(200)
resp.Write([]byte(outputString))
return
} else if outputType == "values" || outputType == "json" {
newOutput := []string{}
for _, key := range newReturn.Keys {
if len(key.Value) == 0 {
continue
}
newOutput = append(newOutput, key.Value)
}
marshalledOutput, err := json.MarshalIndent(newOutput, "", " ")
if err != nil {
log.Printf("[WARNING] Failed to marshal cache values for org %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Something went wrong in cache value json management. Please refresh."}`))
return
}
resp.Header().Set("Content-Type", "application/json")
resp.WriteHeader(200)
resp.Write(marshalledOutput)
return
} else if outputType == "keys" {
fullString := ""
for _, key := range newReturn.Keys {
fullString += fmt.Sprintf("%s\n", key.Key)
}
resp.Write([]byte(fullString))
// Somehow this creates superflous request?
//resp.WriteHeader(200)
return
} else if outputType == "meta" {
marshalledOutput, err := json.MarshalIndent(newReturn.Keys, "", " ")
if err != nil {
log.Printf("[WARNING] Failed to marshal cache keys for org %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Something went wrong in cache key json management. Please refresh."}`))
return
}
resp.Header().Set("Content-Type", "application/json")
resp.WriteHeader(200)
resp.Write(marshalledOutput)
return
}
}
categoryCount, err := GetCacheKeyCount(ctx, orgId, category)
if err != nil {
log.Printf("[WARNING] Failed to get cache key count for org %s: %s", org.Id, err)
} else {
newReturn.TotalAmount = categoryCount
}
b, err := json.Marshal(newReturn)
if err != nil {
log.Printf("[WARNING] Failed to marshal cache keys for org %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Something went wrong in cache key json management. Please refresh."}`))
return
}
if err != nil {
log.Printf("[INFO] Failed getting cache key list for org %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(b)
}
func HandleCacheConfig(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[DEBUG] Api authentication failed in cache config: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
if user.ActiveOrg.Role != "admin" {
log.Printf("[AUDIT] User %s (%s) tried to list cache keys without admin role", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Only admins can distribute cache to sub-orgs"}`))
return
}
var orgId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
orgId = location[4]
}
if len(orgId) == 0 {
log.Printf("[ERROR] Missing org id in cache config")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Missing org id"}`))
return
}
type cacheConfig struct {
Key string `json:"key"`
Action string `json:"action"`
Category string `json:"category"`
SelectedSuborg []string `json:"selected_suborgs"`
}
var config cacheConfig
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
err = json.Unmarshal(body, &config)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling in cache config: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
if config.Category == "default" {
config.Category = ""
}
cacheId := fmt.Sprintf("%s_%s", orgId, config.Key)
cache, err := GetDatastoreKey(ctx, cacheId, config.Category)
if err != nil {
log.Printf("[WARNING] Failed getting cache key '%s' for org %s (config)", config.Key, orgId)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to get key. Does it exist?", "extra": "%s"}`, cache.Key)))
return
}
if config.Action == "suborg_distribute" {
if len(config.SelectedSuborg) == 0 {
cache.SuborgDistribution = []string{}
} else {
cache.SuborgDistribution = config.SelectedSuborg
}
err = SetDatastoreKey(ctx, *cache)
if err != nil {
log.Printf("[WARNING] Failed setting cache key '%s' for org %s (config)", config.Key, orgId)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to set key. Does it exist?", "extra": "%s"}`, cache.Key)))
return
}
}
log.Printf("[INFO] Successfully updated cache key '%s' for org %s", config.Key, orgId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason" : "Cache updated successfully!"}`))
}
func HandleDeleteCacheKey(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[DEBUG] Api authentication failed in delete cache key: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
//for key, value := range data.Apps {
var orgId string
var cacheKey string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
orgId = location[4]
cacheKey = location[6]
}
if len(cacheKey) == 0 || len(orgId) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Missing org id or cache key"}`))
return
}
ctx := GetContext(request)
if orgId != user.ActiveOrg.Id {
log.Printf("[INFO] OrgId '%s' and %s don't match (delete cache key)", orgId, user.ActiveOrg.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Organization ID's don't match"}`))
return
}
cacheKey, err = url.QueryUnescape(strings.Trim(cacheKey, " "))
if err != nil {
log.Printf("[WARNING] Failed to unescape cache key %s: %s", cacheKey, err)
cacheKey = strings.Trim(cacheKey, " ")
}
//cacheKey = strings.Replace(cacheKey, "%20", " ", -1)
cacheKey = strings.Trim(cacheKey, " ")
cacheId := fmt.Sprintf("%s_%s", orgId, cacheKey)
cacheData, err := GetDatastoreKey(ctx, cacheId, "")
if err != nil || cacheData.Key == "" {
log.Printf("[WARNING] Failed to GET datastore key '%s' for org %s (delete)", cacheId, orgId)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to get key. Does it exist?", "extra": "%s"}`, cacheData.Key)))
return
}
if cacheData.OrgId != user.ActiveOrg.Id {
log.Printf("[INFO] OrgId '%s' and '%s' don't match", cacheData.OrgId, user.ActiveOrg.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Organization ID's don't match"}`))
return
}
entity := "org_cache"
DeleteKey(ctx, entity, cacheId)
if len(cacheData.WorkflowId) > 0 {
escapedKey := url.QueryEscape(cacheKey)
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s_%s", orgId, cacheData.WorkflowId, cacheData.Key))
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s_%s", orgId, cacheData.WorkflowId, escapedKey))
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s", cacheData.WorkflowId, cacheData.Key))
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s", cacheData.WorkflowId, escapedKey))
}
DeleteCache(ctx, cacheKey)
DeleteCache(ctx, fmt.Sprintf("datastore_category_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, cacheKey))
DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, orgId))
DeleteCache(ctx, fmt.Sprintf("%s_%s", orgId, cacheData.Key))
DeleteCache(ctx, fmt.Sprintf("%s_%s_%s", orgId, cacheData.Key, cacheData.Category))
if debug {
log.Printf("[DEBUG] Successfully Deleted key '%s' for org %s", cacheKey, orgId)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func HandleDeleteCacheKeyPost(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
//for key, value := range data.Apps {
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
var tmpData CacheKeyData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling in DELETE cache value: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpData.OrgId != fileId {
log.Printf("[INFO] OrgId %s and %s don't match", tmpData.OrgId, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Organization ID's don't match"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[INFO] Organization doesn't exist: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
selectedOrg := tmpData.OrgId
if len(tmpData.ExecutionId) > 0 {
workflowExecution, err := GetWorkflowExecution(ctx, tmpData.ExecutionId)
if err != nil {
log.Printf("[INFO] Failed getting the execution: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No permission to get execution"}`))
return
}
// Allows for execution auth AND user auth
if workflowExecution.Authorization != tmpData.Authorization {
// Get the user?
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
} else {
if user.ActiveOrg.Id != org.Id {
log.Printf("[INFO] Execution auth %s and %s don't match (2)", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
}
}
if workflowExecution.Status != "EXECUTING" {
log.Printf("[INFO] Workflow %s isn't executing and shouldn't be searching", workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow isn't executing (2)"}`))
return
}
if workflowExecution.ExecutionOrg != org.Id {
log.Printf("[INFO] Org %s wasn't used to execute %s", org.Id, workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Bad organization specified"}`))
return
}
} else {
// Fail over to user if exec isn't there
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] Missing auth when deleting key %s for org %s", tmpData.Key, tmpData.OrgId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
if user.ActiveOrg.Id != org.Id {
org, err = GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[INFO] Organization doesn't exist in cache delete: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
selectedOrg = user.ActiveOrg.Id
}
tmpData.Key = strings.Trim(tmpData.Key, " ")
cacheId := fmt.Sprintf("%s_%s", selectedOrg, tmpData.Key)
cacheData, err := GetDatastoreKey(ctx, cacheId, tmpData.Category)
if err != nil || len(cacheData.Key) == 0 {
log.Printf("[ERROR] Failed to DELETE cache key '%s' for org %s (delete) in category '%s'. Does it exist?", tmpData.Key, tmpData.OrgId, tmpData.Category)
resp.WriteHeader(400)
result := ResultChecker{
Success: false,
Reason: "Failed to get key. Does it exist? Correct category?",
Extra: fmt.Sprintf("Attempted to delete key '%s'", tmpData.Key),
}
if len(tmpData.Category) > 0 {
result.Extra = fmt.Sprintf("Attempted to delete key '%s' in category '%s'", tmpData.Key, tmpData.Category)
}
marshalled, err := json.Marshal(result)
if err != nil {
resp.Write([]byte(`{"success": false, "reason": "Failed to get key. Does it exist?"}`))
return
}
resp.Write(marshalled)
return
}
if len(tmpData.Category) > 0 {
cacheId = fmt.Sprintf("%s_%s", cacheId, tmpData.Category)
}
cacheId = url.QueryEscape(cacheId)
if len(cacheId) > 127 {
cacheId = cacheId[:127]
}
if debug {
log.Printf("[DEBUG] Attempting to delete cache key '%s' for org %s. Error: %#v. Cache ID: %s", tmpData.Key, tmpData.OrgId, err, string(cacheId))
}
entity := "org_cache"
err = DeleteKey(ctx, entity, cacheId)
if err != nil {
log.Printf("[WARNING] Failed to DELETE cache key '%s' for org %s (delete) (2)", cacheId, tmpData.OrgId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed to delete key"}`))
return
}
if len(cacheData.WorkflowId) > 0 {
escapedKey := url.QueryEscape(tmpData.Key)
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s_%s", org.Id, cacheData.WorkflowId, cacheData.Key))
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s_%s", org.Id, cacheData.WorkflowId, escapedKey))
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s", cacheData.WorkflowId, cacheData.Key))
DeleteKey(ctx, entity, fmt.Sprintf("%s_%s", cacheData.WorkflowId, escapedKey))
}
DeleteCache(ctx, tmpData.Key)
DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, tmpData.Key))
DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, org.Id))
DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, cacheId))
DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, url.QueryEscape(cacheId)))
normalizedCategory := strings.ReplaceAll(strings.ToLower(tmpData.Category), " ", "_")
if normalizedCategory == "default" {
normalizedCategory = ""
}
DeleteCache(ctx, fmt.Sprintf("%s__%s_%s", entity, org.Id, normalizedCategory))
DeleteCache(ctx, fmt.Sprintf("%s__%s_", entity, org.Id))
DeleteCache(ctx, fmt.Sprintf("%s__%s", entity, org.Id))
result := ResultChecker{
Success: true,
Reason: fmt.Sprintf("Key '%s' deleted", tmpData.Key),
}
if debug {
log.Printf("[DEBUG] Successfully Deleted key '%s' for org %s in category '%s'", tmpData.Key, tmpData.OrgId, tmpData.Category)
}
// Marshal
resp.WriteHeader(200)
jsonResult, err := json.Marshal(result)
if err != nil {
log.Printf("[WARNING] Failed to marshal result: %s", err)
resp.Write([]byte(`{"success": true}`))
return
}
resp.Write([]byte(jsonResult))
}
func HandleGetCacheKey(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
//for key, value := range data.Apps {
var fileId string
location := strings.Split(request.URL.String(), "/")
///api/v2/datastore/category/{category_key}/{key}
///api/v1/orgs/{orgId}/get_cache
///api/v1/get_cache
///api/v1/orgs/{orgId}/datastore/{cache_key}
///api/v1/orgs/{orgId}/cache/{cache_key}
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[ERROR] Path too short: %d", len(location))
fileId = ""
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false}`))
//return
} else {
fileId = location[4]
}
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
// Check if request method is POST
// 3 different auth mechanisms due to public exposing of this endpoint, and for use in workflows
query := request.URL.Query()
requireCacheAuth := false
skipExecutionAuth := false
var tmpData CacheKeyData
if request.Method == "POST" {
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling in GET value: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if tmpData.OrgId != fileId {
if fileId == "" {
fileId = tmpData.OrgId
} else {
log.Printf("[INFO] OrgId %s and %s don't match", tmpData.OrgId, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Organization ID's don't match"}`))
return
}
}
user, err := HandleApiAuthentication(resp, request)
if err == nil {
if len(fileId) == 0 {
fileId = user.ActiveOrg.Id
tmpData.OrgId = user.ActiveOrg.Id
} else {
user.ActiveOrg.Id = fileId
}
skipExecutionAuth = true
if user.ActiveOrg.Id != fileId {
log.Printf("[INFO] OrgId %s and %s don't match in get cache key list. Checking cache auth", user.ActiveOrg.Id, fileId)
requireCacheAuth = true
skipExecutionAuth = false
user.ActiveOrg.Id = fileId
}
}
} else {
if len(location) <= 6 {
log.Printf("[ERROR] Cache Path too short: %d", len(location))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if strings.Contains(location[6], "?") {
location[6] = strings.Split(location[6], "?")[0]
}
// urlescape
parsedCacheKey, err := url.QueryUnescape(location[6])
if err != nil {
log.Printf("[ERROR] Failed to unescape cache key: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
tmpData = CacheKeyData{
OrgId: fileId,
Key: parsedCacheKey,
}
// Use normal user auth
user, usererr := HandleApiAuthentication(resp, request)
if usererr != nil {
// Check if authorization query exists
if len(query.Get("authorization")) == 0 {
log.Printf("[INFO] Failed to authenticate user in GET datastore key: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No authorization provided"}`))
return
}
requireCacheAuth = true
user.ActiveOrg.Id = fileId
}
if user.ActiveOrg.Id != fileId && len(fileId) == 36 {
log.Printf("[INFO] OrgId %s and %s don't match in get cache key list. Checking cache auth", user.ActiveOrg.Id, fileId)
requireCacheAuth = true
user.ActiveOrg.Id = fileId
/*
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Organization ID's don't match"}`))
return
*/
}
// /api/v2/datastore/category/{category_key}/{key}
if tmpData.OrgId == "category" && len(location) == 7 {
tmpData.OrgId = user.ActiveOrg.Id
tmpData.Category = location[5]
tmpData.Key = location[6]
if strings.Contains(tmpData.Key, "?") {
tmpData.Key = strings.Split(tmpData.Key, "?")[0]
}
}
skipExecutionAuth = true
}
ctx := GetContext(request)
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[INFO] Organization '%s' doesn't exist in get cache: %s", tmpData.OrgId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
executionId := ""
if !skipExecutionAuth {
workflowExecution, err := GetWorkflowExecution(ctx, tmpData.ExecutionId)
if err != nil {
log.Printf("[INFO] Failed getting the execution: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "No permission to get execution"}`))
return
}
// Allows for execution auth AND user auth
if workflowExecution.Authorization != tmpData.Authorization {
// Get the user?
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
} else {
if user.ActiveOrg.Id != org.Id {
log.Printf("[INFO] Execution auth %s and %s don't match (2)", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
}
}
if workflowExecution.ExecutionOrg != org.Id {
log.Printf("[INFO] Org %s wasn't used to execute %s", org.Id, workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Bad organization specified"}`))
return
}
/*
if workflowExecution.Status != "EXECUTING" {
log.Printf("[INFO] Workflow %s isn't executing and shouldn't be searching", workflowExecution.ExecutionId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow isn't executing (3)"}`))
return
}
*/
executionId = workflowExecution.ExecutionId
}
//if debug {
// log.Printf("\n\n[DEBUG] Getting key '%s' from category '%s'\n\n", tmpData.Key, tmpData.Category)
//}
tmpData.Key = strings.Trim(tmpData.Key, " ")
cacheId := fmt.Sprintf("%s_%s", tmpData.OrgId, tmpData.Key)
cacheData, err := GetDatastoreKey(ctx, cacheId, tmpData.Category)
if err != nil {
log.Printf("[WARNING] Failed to GET cache key '%s' for org %s (get) and cacheId %s", tmpData.Key, tmpData.OrgId, cacheId)
// Doing a last resort search, e.g. to handle spaces and the like
limit := 50
// HOT FIX FOR UJIMA ALERT
if os.Getenv("SHUFFLE_GCEPROJECT") == "shuffle-europe-west3" {
limit = 2000
}
allkeys, _, err := GetAllCacheKeys(ctx, org.Id, "", limit, "")
if err == nil {
cacheData = &CacheKeyData{}
searchkey := strings.ReplaceAll(strings.Trim(strings.ToLower(tmpData.Key), " "), " ", "_")
for _, key := range allkeys {
tmpkey := strings.ReplaceAll(strings.Trim(strings.ToLower(key.Key), " "), " ", "_")
//log.Printf("%s vs %s", tmpkey, searchkey)
if tmpkey == searchkey {
if debug {
log.Printf("\n\n[DEBUG] Found key %s for org %s\n\n", key.Key, org.Id)
}
cacheData = &key
break
}
}
if cacheData.Key == "" {
log.Printf("[WARNING] Failed to GET datastore key %s for org %s (get)", tmpData.Key, tmpData.OrgId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication or key doesn't exist"}`))
return
}
} else {
log.Printf("[WARNING][%s] Failed to GET datastore key %s for org %s (get)", executionId, tmpData.Key, tmpData.OrgId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication or key doesn't exist"}`))
return
}
}
if len(cacheData.PublicAuthorization) == 0 && cacheData.Category != "protected" {
cacheId := fmt.Sprintf("%s_%s", tmpData.OrgId, tmpData.Key)
if len(tmpData.Category) > 0 && tmpData.Category != "default" {
cacheId = fmt.Sprintf("%s_%s", cacheId, tmpData.Category)
}
cacheId = url.QueryEscape(cacheId)
parsedKey := fmt.Sprintf("org_cache_%s", cacheId)
go DeleteCache(ctx, parsedKey)
}
if requireCacheAuth {
authQuery := query.Get("authorization")
log.Printf("[INFO] Cache auth required for '%s'. Input auth: %s. Required auth: %#v", tmpData.Key, authQuery, cacheData.PublicAuthorization)
if cacheData.PublicAuthorization == "" || authQuery != cacheData.PublicAuthorization {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication or key doesn't exist"}`))
return
}
if cacheData.Category == "protected" {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication or key doesn't exist"}`))
return
}
}
cacheData.Success = true
cacheData.ExecutionId = ""
cacheData.Authorization = ""
cacheData.OrgId = ""
// Look for query param "type"
typeQuery := query.Get("type")
// Check for header accept
if typeQuery == "text" || typeQuery == "raw" || request.Header.Get("Accept") == "text/plain" {
if typeQuery == "text" {
// Check if the value is valid JSON or not
var newstring = ""
var jsonCheck []interface{}
// If it's valid JSON list, add all items to a string with newlines
err := json.Unmarshal([]byte(cacheData.Value), &jsonCheck)
if err == nil {
for _, item := range jsonCheck {
newstring += fmt.Sprintf("%v\n", item)
}
}
if newstring != "" {
cacheData.Value = newstring
}
}
resp.Header().Set("Content-Type", "text/plain")
resp.WriteHeader(200)
resp.Write([]byte(cacheData.Value))
return
} else if typeQuery == "json" {
resp.Header().Set("Content-Type", "application/json")
//validate if it's json or not
isValidJson := false
cacheData.Value = strings.Trim(cacheData.Value, " ")
if strings.HasPrefix(cacheData.Value, "{") && strings.HasSuffix(cacheData.Value, "}") || strings.HasPrefix(cacheData.Value, "[") && strings.HasSuffix(cacheData.Value, "]") {
// Check if it's a list of JSON
listMarshalled := []interface{}{}
err := json.Unmarshal([]byte(cacheData.Value), &listMarshalled)
if err == nil {
isValidJson = true
outputBody, err := json.MarshalIndent(listMarshalled, "", " ")
if err == nil {
cacheData.Value = string(outputBody)
}
} else {
objectMarshalled := map[string]interface{}{}
err := json.Unmarshal([]byte(cacheData.Value), &objectMarshalled)
if err == nil {
isValidJson = true
outputBody, err := json.MarshalIndent(objectMarshalled, "", " ")
if err == nil {
cacheData.Value = string(outputBody)
}
} else {
//log.Printf("[INFO] Cache key %s for org %s isn't valid JSON: '%s'", tmpData.Key, tmpData.OrgId, cacheData.Value)
isValidJson = false
}
}
}
if !isValidJson {
jsonlist := []string{}
if strings.Contains(cacheData.Value, "\n") {
if strings.Count(cacheData.Value, "\n") == 1 {
if strings.Contains(cacheData.Value, ",") {
jsonlist = strings.Split(cacheData.Value, ",")
} else {
jsonlist = strings.Split(cacheData.Value, "\n")
}
} else {
jsonlist = strings.Split(cacheData.Value, "\n")
}
}
parsedJsonlist, err := json.MarshalIndent(jsonlist, "", " ")
if err != nil {
log.Printf("[WARNING] Failed to parse JSON list for key %s for org %s", tmpData.Key, tmpData.OrgId)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to parse JSON list"}`))
return
}
cacheData.Value = string(parsedJsonlist)
}
resp.WriteHeader(200)
resp.Write([]byte(cacheData.Value))
return
}
b, err := json.Marshal(cacheData)
if err != nil {
log.Printf("[WARNING] Failed to marshal cache data %s for org %s", tmpData.Key, tmpData.OrgId)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to get key. Does it exist?"}`))
return
}
resp.WriteHeader(200)
resp.Write(b)
}
func HandleSetDatastoreKey(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body in set cache: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading set datastore key body"}`))
return
}
var tmpData []CacheKeyData
err = json.Unmarshal(body, &tmpData)
if err != nil {
var tmpDataOverride CacheKeyDataFallback
err = json.Unmarshal(body, &tmpDataOverride)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling in setvalue (1): %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Check if value is a map[] or []map first
parsedValue := ""
if _, ok := tmpDataOverride.Value.(string); ok {
parsedValue = tmpDataOverride.Value.(string)
} else {
marshalledValue, err := json.Marshal(tmpDataOverride.Value)
if err == nil {
parsedValue = string(marshalledValue)
} else {
log.Printf("[WARNING] Failed to marshal value in setvalue: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed to parse value. Make sure it is in the [{"key": "key", "value": "value"}] format."}`))
return
}
}
tmpData = append(tmpData, CacheKeyData{
OrgId: tmpDataOverride.OrgId,
Key: tmpDataOverride.Key,
Category: tmpDataOverride.Category,
Tags: tmpDataOverride.Tags,
Enrichments: tmpDataOverride.Enrichments,
Value: parsedValue,
})
}
if len(tmpData) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No data provided. Value of each key should be a string."}`))
return
}
ctx := GetContext(request)
user, usererr := HandleApiAuthentication(resp, request)
if usererr != nil || len(user.ActiveOrg.Id) == 0 {
sourceExecution, sourceExecutionOk := request.URL.Query()["execution_id"]
sourceAuth, sourceAuthOk := request.URL.Query()["authorization"]
if !sourceAuthOk || !sourceExecutionOk {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication (1)"}`))
return
}
foundExec, err := GetWorkflowExecution(ctx, sourceExecution[0])
if err != nil {
log.Printf("[WARNING] Failed getting exec during cache set: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "No permission to get execution (2)"}`))
return
}
if sourceAuth[0] != foundExec.Authorization {
log.Printf("[INFO] Execution auth %s and %s don't match", foundExec.Authorization, sourceAuth[0])
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication (3)"}`))
return
}
if len(foundExec.ExecutionOrg) == 0 {
log.Printf("[WARNING] Execution %s doesn't have an org set", foundExec.ExecutionId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication (4)"}`))
return
}
user.ActiveOrg.Id = foundExec.ExecutionOrg
}
mainCategory := ""
for itemIndex, _ := range tmpData {
tmpData[itemIndex].UpdatedBy = user.Username
tmpData[itemIndex].OrgId = user.ActiveOrg.Id
mainCategory = tmpData[itemIndex].Category
if strings.ToLower(tmpData[itemIndex].Category) == "default" {
tmpData[itemIndex].Category = ""
}
}
log.Printf("[AUDIT] Running bulk upload for org %s to category '%s. Keys: %d'. Tags: %#v", user.ActiveOrg.Id, mainCategory, len(tmpData), tmpData[0].Tags)
existingInfo, err := SetDatastoreKeyBulk(ctx, tmpData)
if err != nil {
log.Printf("[ERROR] Failed to set %d datastore key(s) for org %s", len(tmpData), user.ActiveOrg.Id)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to set data. Please try again, or contact support@shuffler.io"}`))
return
}
log.Printf("[INFO] Successfully set %d datastore keys (or less) for org '%s' (%s)", len(tmpData), user.ActiveOrg.Name, user.ActiveOrg.Id)
type returnStruct struct {
Success bool `json:"success"`
KeysExisted []DatastoreKeyMini `json:"keys_existed"`
}
/*
// For testing deduplication
if debug {
found := []string{}
for _, existing := range existingInfo {
if ArrayContains(found, existing.Key) {
log.Printf("[DEBUG] Key %s already found in existing info", existing.Key)
continue
}
found = append(found, existing.Key)
}
}
*/
returnData := returnStruct{
Success: true,
KeysExisted: existingInfo,
}
b, err := json.Marshal(returnData)
if err != nil {
log.Printf("[ERROR] Failed to marshal return data in set datastore key. Setting to JUST success true. This should NEVER happen. Details: %s", err)
b = []byte(`{"success": true}`)
}
resp.WriteHeader(200)
resp.Write(b)
}
func HandleSetCacheKey(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, usererr := HandleApiAuthentication(resp, request)
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body in set cache: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
//for key, value := range data.Apps {
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
// Check if body contains "key": and replace it, as it should be a string
var tmpData CacheKeyDataMini
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling in setvalue (2): %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
if len(tmpData.OrgId) == 0 {
//log.Printf("[INFO] No org id specified. User org: %#v", user.ActiveOrg)
tmpData.OrgId = user.ActiveOrg.Id
}
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[WARNING] Organization doesn't exist: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
workflowExecution, err := GetWorkflowExecution(ctx, tmpData.ExecutionId)
if err != nil {
if len(tmpData.ExecutionId) > 0 {
log.Printf("[WARNING] Failed getting exec during cache set: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "No permission to get execution"}`))
return
}
workflowExecution.Authorization = uuid.NewV4().String()
}
if workflowExecution.Authorization != tmpData.Authorization || len(tmpData.Authorization) == 0 || len(workflowExecution.Authorization) == 0 {
// Get the user?
if usererr != nil {
log.Printf("[INFO] Execution auth %s and %s don't match", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
} else {
if user.ActiveOrg.Id != org.Id {
log.Printf("[INFO] Execution auth %s and %s don't match (2)", workflowExecution.Authorization, tmpData.Authorization)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
tmpData.OrgId = user.ActiveOrg.Id
}
} else {
if workflowExecution.Status != "EXECUTING" {
log.Printf("[INFO] Workflow '%s' isn't executing and shouldn't be searching", workflowExecution.ExecutionId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow isn't executing (4)"}`))
return
}
if workflowExecution.ExecutionOrg != org.Id {
log.Printf("[INFO] Org '%s' wasn't used to execute %s", org.Id, workflowExecution.ExecutionId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Bad organization specified"}`))
return
}
}
if tmpData.OrgId != fileId {
log.Printf("[INFO] OrgId '%s' and '%s' don't match (set cache)", tmpData.OrgId, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Organization ID's don't match"}`))
return
}
if len(tmpData.Value) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Value can't be empty"}`))
return
}
if strings.ToLower(tmpData.Category) == "default" {
tmpData.Category = ""
}
tmpData.Key = strings.Trim(tmpData.Key, " ")
// Check if cache already existed and if distributed
cacheId := fmt.Sprintf("%s_%s", tmpData.OrgId, tmpData.Key)
cacheData, err := GetDatastoreKey(ctx, cacheId, tmpData.Category)
if err == nil {
tmpData.SuborgDistribution = cacheData.SuborgDistribution
}
// This is just to ensure that input data doesn't get directly set in the database
parsedKey := CacheKeyData{
Category: tmpData.Category,
Key: tmpData.Key,
Value: tmpData.Value,
ExecutionId: tmpData.ExecutionId,
Authorization: tmpData.Authorization,
SuborgDistribution: tmpData.SuborgDistribution,
Tags: tmpData.Tags,
IgnoreSecurityRules: tmpData.IgnoreSecurityRules, // Makes sure we don't stop manual requests even if security rules exist. Basically a rule.
OrgId: user.ActiveOrg.Id,
UpdatedBy: user.Username,
}
if len(user.ActiveOrg.Id) == 0 {
parsedKey.OrgId = tmpData.OrgId
}
existed, err := SetDatastoreKeyBulk(ctx, []CacheKeyData{parsedKey})
if err != nil {
log.Printf("[ERROR] Failed to set cache key '%s' for org %s", tmpData.Key, tmpData.OrgId)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to set data. Please try again, or contact support@shuffler.io"}`))
return
}
if len(existed) == 0 {
log.Printf("[INFO] Successfully set key '%s' for org '%s' (%s). Category: %s", tmpData.Key, org.Name, tmpData.OrgId, tmpData.Category)
} else {
log.Printf("[INFO] Successfully set key '%s' for org '%s' (%s). New key: %#v. Category: %s", tmpData.Key, org.Name, tmpData.OrgId, !existed[0].Existed, tmpData.Category)
}
type returnStruct struct {
Success bool `json:"success"`
KeysExisted []DatastoreKeyMini `json:"keys_existed"`
}
returnData := returnStruct{
Success: true,
KeysExisted: existed,
}
b, err := json.Marshal(returnData)
if err != nil {
b = []byte(`{"success": true}`)
}
resp.WriteHeader(200)
resp.Write(b)
}
// Checks authentication string for Webhooks
func CheckHookAuth(request *http.Request, auth string) error {
if len(auth) == 0 {
return nil
}
authSplit := strings.Split(auth, "\n")
for _, line := range authSplit {
lineSplit := strings.Split(line, "=")
if strings.Contains(line, ":") {
lineSplit = strings.Split(line, ":")
}
if len(lineSplit) >= 2 {
validationHeader := strings.ToLower(strings.TrimSpace(lineSplit[0]))
found := false
joinedItemValue := strings.Join(lineSplit[1:], "=")
// log.Printf("[INFO] Checking %s = %s", validationHeader, joinedItemValue)
for key, value := range request.Header {
if strings.ToLower(key) == validationHeader && len(value) > 0 {
//log.Printf("FOUND KEY %s. Value: %s", validationHeader, value)
if value[0] == strings.TrimSpace(joinedItemValue) {
found = true
break
}
}
}
if !found {
return errors.New(fmt.Sprintf("Missing or bad header: %s", validationHeader))
}
//log.Printf("Find header %s", validationHeader)
//itemHeader := request.Header[validationHeader]
//log.Printf("LINE: %s. Header: %s", line, itemHeader)
} else {
log.Printf("[WARNING] Bad auth line: %s. NOT checking auth.", line)
}
}
//return errors.New("Bad auth!")
return nil
}
// Body = The action body received from the user to test.
func PrepareSingleAction(ctx context.Context, parentRequest *http.Request, user User, appId string, body []byte, runValidationAction bool, decision ...string) (WorkflowExecution, error) {
workflowExecution := WorkflowExecution{}
if ctx == nil {
ctx = context.Background()
}
var action Action
err := json.Unmarshal(body, &action)
if err != nil {
log.Printf("[WARNING] Failed action single execution unmarshaling: %s", err)
return workflowExecution, err
}
if debug {
log.Printf("[DEBUG] Action: %#v (%s)", action.Name, action.AppID)
}
if appId != action.AppID {
// Used for standalone runs controlled from /agents and /mcp
if appId == "agent_starter" {
workflowId := uuid.NewV4().String()
action.SourceWorkflow = workflowId
if len(action.ID) != 36 {
action.ID = uuid.NewV4().String()
}
exec := WorkflowExecution{
Workflow: Workflow{
ID: workflowId,
Actions: []Action{
action,
},
OrgId: user.ActiveOrg.Id,
Owner: user.Username,
UpdatedBy: user.Username,
Start: action.ID,
},
Type: "AGENT",
Start: action.ID,
Status: "EXECUTING",
WorkflowId: workflowId,
ExecutionId: workflowId,
ExecutionOrg: user.ActiveOrg.Id,
StartedAt: int64(time.Now().Unix()),
Authorization: uuid.NewV4().String(),
}
SetWorkflowExecution(ctx, exec, true)
if os.Getenv("AGENT_TEST_MODE") == "true" {
var bodyMap map[string]interface{}
if err := json.Unmarshal(body, &bodyMap); err == nil {
if mockToolCalls, ok := bodyMap["mock_tool_calls"]; ok {
mockCacheKey := fmt.Sprintf("agent_mock_%s", exec.ExecutionId)
mockData, _ := json.Marshal(mockToolCalls)
err := SetCache(ctx, mockCacheKey, mockData, 10)
if err != nil {
log.Printf("[ERROR] Failed to set cache the mock_tool_calls data for the exection %s", exec.ExecutionId)
}
log.Printf("[DEBUG] Cached mock tool calls for execution %s", exec.ExecutionId)
} else {
log.Printf("[WARNING] No mock_tool_calls found in the request body")
}
}
}
callerName := "PrepareSingleAction"
action, err := HandleAiAgentExecutionStart(exec, action, false, callerName)
if err != nil {
log.Printf("[ERROR] Failed to handle AI agent execution start: %s", err)
}
exec.Workflow.Actions[0] = action
newExec, err := GetWorkflowExecution(ctx, exec.ExecutionId)
log.Printf("[INFO][%s] AI Agent: %s Started standalone for org %s, execution id %s, workflow %s", exec.ExecutionId, callerName, user.ActiveOrg.Id, exec.ExecutionId, exec.WorkflowId)
if err != nil {
log.Printf("[ERROR] Failed to get workflow execution after starting agent: %s", err)
} else {
return *newExec, nil
}
return exec, nil
} else if appId == "agent" {
// This is used in HandleAiAgentExecutionStart as a way to
// run a single action with a specific appId
// Also used for rerunning a single decision within an existing shuffle_agent execution on the /agents page.
action.AppID = "agent"
} else if strings.ToLower(appId) == "http" || strings.ToLower(action.AppID) == "http" {
action.AppID = "http"
} else {
log.Printf("[WARNING] Bad appid in single execution of App '%s'", appId)
return workflowExecution, errors.New(fmt.Sprintf("No matching app found for '%s'. Did you choose an app to run?", appId))
}
}
if len(action.ID) == 0 {
action.ID = uuid.NewV4().String()
}
if len(action.Name) == 0 {
return workflowExecution, errors.New("Action name can't be empty")
}
app := WorkflowApp{}
decisionId := ""
if strings.ToLower(appId) == "agent" {
if len(decision) > 0 {
decisionId = decision[0]
}
} else if strings.ToLower(appId) == "integration" || strings.ToLower(appId) == "singul" {
log.Printf("[INFO] Running single action for 'integration' app => Singul")
// Related to sensor groups for Orborus
} else if strings.ToLower(appId) == "sensors" && action.Name == "run_action" {
if len(user.ActiveOrg.Id) == 0 {
return workflowExecution, errors.New("No org ID supplied for sensor execution")
}
if user.Role != "admin" {
return workflowExecution, errors.New("Command execution requires Org Admin access")
}
log.Printf("[INFO] Running Shuffle Group sensor action for org '%s'", user.ActiveOrg.Id)
foundHosts := []string{}
foundAction := ""
foundSensorGroup := ""
foundError := ""
for _, param := range action.Parameters {
if len(param.Value) == 0 {
foundError = fmt.Sprintf("'%s' can't be empty. Required fields: action, hosts, sensor_group", param.Name)
break
}
if param.Name == "hosts" {
foundHosts = strings.Split(param.Value, ",")
} else if param.Name == "action" {
foundAction = param.Value
} else if param.Name == "sensor_group" {
foundSensorGroup = param.Value
}
}
if len(foundError) > 0 {
return workflowExecution, errors.New(foundError)
}
foundExec := workflowExecution
foundEnv := foundSensorGroup
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
return foundExec, err
}
parsedEnv := ""
for _, env := range environments {
if env.Archived {
continue
}
if !env.SensorGroup {
continue
}
if env.Name != foundEnv {
continue
}
parsedEnv = fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(foundEnv, " ", "-"), "_", "-")), env.OrgId)
break
}
if len(parsedEnv) == 0 {
return foundExec, errors.New("Failed to find environment for sensor action. Make sure the environment exists and isn't archived.")
}
// One for each host
go IncrementCache(ctx, user.ActiveOrg.Id, "app_executions", len(foundHosts))
go IncrementCache(ctx, user.ActiveOrg.Id, "workflow_executions", len(foundHosts))
for hostIndex, host := range foundHosts {
workflowId := uuid.NewV4().String()
action.SourceWorkflow = workflowId
if len(action.ID) != 36 {
action.ID = uuid.NewV4().String()
}
startTime := int64(time.Now().Unix())
exec := WorkflowExecution{
Workflow: Workflow{
ID: workflowId,
Actions: []Action{
action,
},
OrgId: user.ActiveOrg.Id,
Owner: user.Username,
UpdatedBy: user.Username,
Start: action.ID,
},
Type: "SENSOR_ACTION",
Start: action.ID,
Status: "EXECUTING",
WorkflowId: workflowId,
ExecutionId: workflowId,
ExecutionOrg: user.ActiveOrg.Id,
StartedAt: startTime,
Authorization: uuid.NewV4().String(),
}
if hostIndex == 0 {
foundExec = exec
}
go SetWorkflowExecution(ctx, exec, true)
executionRequest := ExecutionRequest{
Start: exec.Start,
ExecutionId: exec.ExecutionId,
Authorization: exec.Authorization,
WorkflowId: exec.Workflow.ID,
Environments: []string{parsedEnv},
Type: "SENSOR_ACTION",
Priority: 5,
ExecutionArgument: foundAction,
ExecutionSource: host,
}
// Queue logging to make sure we get it
log.Printf("[INFO][%s] Queued SENSOR_ACTION to be ran on hosts %s in group %s", exec.ExecutionId, foundHosts, foundEnv)
err = SetWorkflowQueue(ctx, executionRequest, parsedEnv)
if err != nil {
log.Printf("[WARNING][%s] Failed adding %s to db (single action queue): %s", exec.ExecutionId, parsedEnv, err)
}
}
return foundExec, nil
} else if strings.ToLower(appId) == "http" {
// Find the app and the ID for it
apps, err := FindWorkflowAppByName(ctx, "http")
if err != nil {
log.Printf("[WARNING] Failed to find HTTP app in single action execution: %s", err)
return workflowExecution, err
} else {
if len(apps) > 0 {
// Just assuming we can use #1
// Find the highest version
app = apps[0]
latestVersion := ""
for _, innerApp := range apps {
// Semver check
if len(innerApp.AppVersion) == 0 {
continue
}
if len(latestVersion) == 0 {
latestVersion = innerApp.AppVersion
app = innerApp
continue
}
v2, err := semver.NewVersion(innerApp.AppVersion)
if err != nil {
log.Printf("[ERROR] Failed parsing original app version %s: %s", innerApp.AppVersion, err)
continue
}
appConstraint := fmt.Sprintf("> %s", latestVersion)
c, err := semver.NewConstraint(appConstraint)
if err != nil {
log.Printf("[ERROR] Failed preparing constraint %s: %s", appConstraint, err)
continue
}
if c.Check(v2) {
app = innerApp
latestVersion = innerApp.AppVersion
}
}
appId = app.ID
action.AppID = app.ID
action.AppVersion = app.AppVersion
action.Label = fmt.Sprintf("HTTP standalone action")
} else {
log.Printf("[WARNING] Failed to find HTTP app in single action execution")
return workflowExecution, errors.New("Failed to find HTTP app. Is it installed?")
}
}
// Check if incoming action is "custom_action" and map it to HTTP
if action.Name == "custom_action" || action.Name == "Custom Action" {
urlIndex := -1
path := ""
queries := ""
for paramIndex, param := range action.Parameters {
if strings.ToLower(param.Name) == "method" {
action.Name = strings.ToUpper(param.Value)
} else if strings.ToLower(param.Name) == "url" {
urlIndex = paramIndex
} else if strings.ToLower(param.Name) == "path" {
path = param.Value
} else if strings.ToLower(param.Name) == "queries" {
queries = param.Value
}
}
if len(path) > 0 && urlIndex >= 0 {
if strings.HasPrefix(path, "/") {
path = path[1:]
}
action.Parameters[urlIndex].Value = fmt.Sprintf("%s/%s", action.Parameters[urlIndex].Value, path)
}
if len(queries) > 0 && urlIndex >= 0 {
// Split them and add to the URL
if strings.Contains(action.Parameters[urlIndex].Value, "?") {
action.Parameters[urlIndex].Value = fmt.Sprintf("%s&%s", action.Parameters[urlIndex].Value, queries)
} else {
action.Parameters[urlIndex].Value = fmt.Sprintf("%s?%s", action.Parameters[urlIndex].Value, queries)
}
}
log.Printf("URL: %#v", action.Parameters[urlIndex].Value)
}
} else {
// Integration handler as it has no name
//action.AppName = "Shuffle-AI"
//action.Name = "run_schemaless"
newApp, err := GetApp(ctx, appId, user, false)
if err != nil || len(newApp.ID) == 0 {
log.Printf("[WARNING] Error getting app (execute SINGLE app action): %s", appId)
return workflowExecution, err
}
if len(action.AppName) == 0 {
action.AppName = newApp.Name
}
if len(action.AppVersion) == 0 {
action.AppVersion = newApp.AppVersion
}
if len(action.AppID) == 0 {
action.AppID = newApp.ID
}
if len(action.Label) == 0 {
action.Label = "Single action"
}
if len(action.Parameters) == 0 {
//action.Parameters = newApp.Parameters
log.Printf("[INFO] No parameters in single action. Does it matter?")
}
app = *newApp
}
// This is NOT a good solution, but a good bypass
if app.Authentication.Required || len(action.AuthenticationId) > 0 {
if len(action.AuthenticationId) > 0 {
if debug {
log.Printf("[DEBUG][%s] Found auth ID for single action: %s", workflowExecution.ExecutionId, action.AuthenticationId)
}
} else {
authFields := 0
foundFields := []string{}
for _, actionParam := range action.Parameters {
if actionParam.Configuration {
authFields += 1
}
foundFields = append(foundFields, strings.ToLower(actionParam.Name))
}
// Usually url
if authFields <= 2 && app.Generated {
// These are ok to append no matter what due to
// cleanup happening in the app run itself anyway. AKA kwargs
// that are unnecessary are not being used
if !ArrayContains(foundFields, "apikey") {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "apikey",
Configuration: true,
})
}
if !ArrayContains(foundFields, "access_token") {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "access_token",
Configuration: true,
})
}
if !ArrayContains(foundFields, "username_basic") {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "username_basic",
Configuration: true,
})
}
if !ArrayContains(foundFields, "password_basic") {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "password_basic",
Configuration: true,
})
}
}
auths, err := GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting auth for single action: %s", err)
} else {
latestTimestamp := int64(0)
for _, auth := range auths {
if auth.App.ID != appId {
if auth.App.Name != app.Name {
continue
}
}
// Fallback to latest created
if latestTimestamp < auth.Edited {
latestTimestamp = auth.Edited
action.AuthenticationId = auth.Id
}
// If valid, just choose it
if auth.Validation.Valid {
action.AuthenticationId = auth.Id
break
}
}
}
}
}
if runValidationAction {
log.Printf("\n\n[INFO] SHOULD BE Running validation action for %s for org %s (%s)\n\n", app.Name, user.ActiveOrg.Name, user.ActiveOrg.Id)
// Find the action tagged to be used for validation:
// 1. Find the action in the app
// 2. Find a GET request that is labeled without required parameters
// 3. Run the default request that is sent in IF possible
// Validation of the workflow will say whether it was successful or not
//for _, appAction := range app.Actions {
//}
}
// Auth is handled in PrepareWorkflowExec, so this may not be needed
newParams := []WorkflowAppActionParameter{}
originalUrl := ""
for _, param := range action.Parameters {
newName := GetValidParameters([]string{param.Name})
if len(newName) > 0 {
param.Name = newName[0]
}
if strings.ToLower(param.Name) == "url" {
originalUrl = param.Value
}
newParams = append(newParams, param)
}
action.Parameters = newParams
action.Sharing = app.Sharing
action.Public = app.Public
action.Generated = app.Generated
if len(action.Environment) == 0 {
if project.Environment == "cloud" {
action.Environment = "cloud"
} else {
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting environments for org in single action %s: %s", user.ActiveOrg.Id, err)
}
for _, env := range environments {
if env.Default {
//log.Printf("[INFO] Setting default environment for single action: %s", env.Name)
action.Environment = env.Name
break
}
}
}
}
action.AppID = appId
workflow := Workflow{
Start: action.ID,
ID: uuid.NewV4().String(),
Generated: true,
Hidden: true,
}
// Make a fake request object as it's not necessary
if user.ActiveOrg.Id != "" {
workflow.Owner = user.Id
workflow.OrgId = user.ActiveOrg.Id
workflow.ExecutingOrg = user.ActiveOrg
workflowExecution.ExecutionOrg = user.ActiveOrg.Id
workflowExecution.OrgId = user.ActiveOrg.Id
}
if len(app.Name) == 0 && len(action.AppName) > 0 {
app.Name = action.AppName
}
if len(app.ID) == 0 && len(action.AppID) > 0 {
app.ID = action.AppID
}
// Prevents overwriting of URL if auth injection is done
shuffleAuthInjected := false
// Fallback to inject creds if the user don't have any. This is for internal +
// AI oriented APIs only. Check IsShuffleApp() for details
isShuffleApp := IsShuffleApp(app)
if isShuffleApp && app.Generated && len(workflowExecution.OrgId) > 0 && len(action.AuthenticationId) == 0 && strings.ToLower(app.Name) != "openai" && strings.ToLower(action.Environment) == "cloud" {
shuffleAuthInjected = true
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 && strings.Contains(os.Getenv("SHUFFLE_CLOUDRUN_URL"), "http") {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
if len(user.ApiKey) == 0 && len(user.Username) > 0 && len(user.Id) > 0 {
newUserInfo, err := GenerateApikey(ctx, user)
if err != nil {
log.Printf("[ERROR] Failed to realtime generate apikey for user %s: %s", user.Username, err)
} else {
user = newUserInfo
}
}
foundApikey := ""
if len(user.ApiKey) > 0 {
foundApikey = user.ApiKey
} else {
org, err := GetOrg(ctx, workflowExecution.OrgId)
if err == nil {
selectedUser := &User{}
for _, curuser := range org.Users {
if len(curuser.Username) == 0 || len(curuser.Id) == 0 {
continue
}
user, err := GetUser(ctx, curuser.Id)
if err != nil {
continue
}
if curuser.Role == "admin" {
selectedUser = user
}
if len(user.ApiKey) > 0 {
foundApikey = user.ApiKey
break
}
}
// In case of further user-issues...
if len(foundApikey) == 0 {
if len(selectedUser.ApiKey) > 0 {
foundApikey = selectedUser.ApiKey
} else {
newUserInfo, err := GenerateApikey(ctx, *selectedUser)
if err != nil {
log.Printf("[ERROR] Failed generating apikey for %s (%s)", selectedUser.Username, selectedUser.Id)
} else {
foundApikey = newUserInfo.ApiKey
}
}
}
} else {
if debug {
log.Printf("[ERROR] Bad org issue in auto-auth mapping: '%s'", workflowExecution.OrgId)
}
}
}
apikeyFound := false
urlFound := false
orgIdFound := false
headerIndex := -1
for paramIndex, param := range action.Parameters {
if param.Name == "apikey" {
action.Parameters[paramIndex].Value = foundApikey
apikeyFound = true
} else if param.Name == "url" {
action.Parameters[paramIndex].Value = backendUrl
action.Parameters[paramIndex].Example = backendUrl
urlFound = true
} else if param.Name == "orgid" {
action.Parameters[paramIndex].Value = workflowExecution.OrgId
orgIdFound = true
} else if param.Name == "headers" {
headerIndex = paramIndex
}
}
if !apikeyFound {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "apikey",
Value: foundApikey,
})
}
if !urlFound {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "url",
Value: backendUrl,
Example: backendUrl,
})
}
if !orgIdFound {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "orgid",
Value: workflowExecution.OrgId,
})
}
if headerIndex == -1 && len(workflowExecution.OrgId) > 0 {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "headers",
Value: fmt.Sprintf("Org-Id: %s", workflowExecution.OrgId),
})
} else {
action.Parameters[headerIndex].Value = fmt.Sprintf("%s\nOrg-Id: %s", action.Parameters[headerIndex].Value, workflowExecution.OrgId)
}
// Custom AI injection when necessary
} else if strings.ToLower(app.Name) == "openai" && len(action.AuthenticationId) == 0 {
shuffleAuthInjected = true
// cloud => only do it on cloud location
// This prevents local users from being able to see it
if project.Environment != "cloud" || (project.Environment == "cloud" && strings.ToLower(action.Environment) == "cloud") {
apiKey := os.Getenv("AGENT_LLM_API_KEY")
if apiKey == "" {
apiKey = os.Getenv("AI_API_KEY")
}
if apiKey == "" {
apiKey = os.Getenv("OPENAI_API_KEY")
}
apiUrl := os.Getenv("AGENT_LLM_API_URL")
if apiUrl == "" {
apiUrl = os.Getenv("AI_API_URL")
}
if apiUrl == "" {
apiUrl = os.Getenv("OPENAI_API_URL")
}
if apiUrl == "" {
apiUrl = "https://api.openai.com"
}
// Only inject system credentials if this request came from a legitimate agent execution. The one-time token is set by HandleAiAgentExecutionStart, consumed here on first use, and cannot be replayed.
agentTokenValid := false
if parentRequest != nil {
agentToken := parentRequest.Header.Get("X-Agent-Token")
if len(agentToken) > 0 {
agentTokenCacheKey := fmt.Sprintf("agent_onetime_token_%s", agentToken)
if cachedVal, err := GetCache(ctx, agentTokenCacheKey); err == nil && cachedVal != nil {
go DeleteCache(ctx, agentTokenCacheKey)
agentTokenValid = true
}
}
}
if len(apiKey) > 0 && agentTokenValid {
IncrementCache(ctx, user.ActiveOrg.Id, "ai_executions", 1)
urlFound := false
apikeyFound := false
for paramIndex, param := range action.Parameters {
// Cleanup to prevent bad functions from being injected
// Such as liquid
if project.Environment == "cloud" && strings.ToLower(action.Environment) == "cloud" {
action.Parameters[paramIndex].Value = strings.ReplaceAll(action.Parameters[paramIndex].Value, "${", "")
action.Parameters[paramIndex].Value = strings.ReplaceAll(action.Parameters[paramIndex].Value, "{{", "")
action.Parameters[paramIndex].Value = strings.ReplaceAll(action.Parameters[paramIndex].Value, "python", "")
}
if param.Name == "url" {
action.Parameters[paramIndex].Value = apiUrl
urlFound = true
} else if param.Name == "apikey" {
action.Parameters[paramIndex].Value = apiKey
action.Parameters[paramIndex].Configuration = true
apikeyFound = true
}
}
if !urlFound {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "url",
Value: apiUrl,
})
}
if !apikeyFound {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "apikey",
Value: apiKey,
Configuration: true,
})
}
//log.Printf("[AUDIT] Injected system AI credentials (fallback) for org %s", user.ActiveOrg.Id)
// Mapping to internal so the execution itself is not referencable
// FIXME: This doesn't work well, so we're just filtering out these
// executions until FINISHED (AKA cleaned up)
if project.Environment == "cloud" {
workflow.ID = "INTERNAL"
workflow.OrgId = "INTERNAL"
workflow.ExecutingOrg = OrgMini{
Name: "INTERNAL",
Id: "INTERNAL",
}
workflowExecution.Workflow = workflow
workflowExecution.ExecutionOrg = "INTERNAL"
workflowExecution.Workflow.OrgId = "INTERNAL"
}
}
}
}
// Used for very deep recursion testing of specific injections - e.g. for
// agents -> singul -> agents -> singul -> app
/*
if debug {
log.Printf("APP: %s. Org: %#v", action.AppName, workflowExecution.OrgId)
marshalledActions, _ := json.MarshalIndent(action.Parameters, "", " ")
log.Printf("ACTION PARAMS:\n%s", string(marshalledActions))
if action.AppName != "AI Agent" && action.AppName != "openai" {
//os.Exit(3)
}
}
*/
workflow.Actions = []Action{
action,
}
// Add fake queries to it. Doesn't matter what is here.
// This is just to ensure that _something_ is sent
badRequest := &http.Request{}
badRequest.URL, _ = url.Parse(fmt.Sprintf("http://localhost:3000/api/v1/workflows/%s/execute", workflow.ID))
badRequest.URL.RawQuery = fmt.Sprintf("")
badRequest.Method = "GET"
workflowExecution, _, errString, err := PrepareWorkflowExecution(ctx, workflow, badRequest, 10)
if err != nil || len(errString) > 0 {
// FIXME: Handle other error returns as well?
if strings.Contains(errString, "App Auth ID") {
log.Printf("[DEBUG] Bad auth ID provided for single action: %s", errString)
return workflowExecution, errors.New("The authentication ID provided is invalid. Please try another.")
}
log.Printf("[ERROR] Failed preparing single execution (%s): %s", workflowExecution.ExecutionId, err)
}
if len(action.SourceWorkflow) > 0 {
if len(action.ID) == 0 {
return workflowExecution, errors.New("No action ID provided. This is required for Action reruns to deduplicate results.")
}
if len(action.SourceExecution) == 0 {
return workflowExecution, errors.New("No source_execution provided")
}
workflow, err := GetWorkflow(ctx, action.SourceWorkflow, true)
if err != nil {
return workflowExecution, err
}
if workflow.OrgId != user.ActiveOrg.Id && len(workflow.ID) > 0 {
return workflowExecution, errors.New(fmt.Sprintf("Workflow doesn't belong to the same organization (%s vs %s)", workflow.OrgId, user.ActiveOrg.Id))
}
// Check if the execution exists
workflowExecution.WorkflowId = workflow.ID
oldExec, err := GetWorkflowExecution(ctx, action.SourceExecution)
if err != nil {
return workflowExecution, err
}
if oldExec.Workflow.ID != action.SourceWorkflow {
return workflowExecution, errors.New("Previous execution (source_execution) doesn't belong to the workflow. Please try again.")
}
// Updated action stuff, ensuring everything is on par
if len(workflowExecution.Workflow.Actions) == 1 {
action = workflowExecution.Workflow.Actions[0]
}
// Fill in missing actions and dedup
foundResultIndex := -1
action.Category = "rerun"
newResults := []ActionResult{}
for resIndex, result := range oldExec.Results {
if result.Action.ID == action.ID {
foundResultIndex = resIndex
continue
}
foundIndex := -1
for foundResultIndex, foundResult := range workflowExecution.Results {
if foundResult.Action.ID == result.Action.ID {
foundIndex = foundResultIndex
newResults = append(newResults, foundResult)
break
}
}
if foundIndex == -1 {
// This is to KNOW that it's a rerun.
// Just had to use an existing field, as we don't wanna keep bloating the struct
result.Action.Category = "rerun"
// Ensures "normal" behavior based on existing data
if result.Status != "SKIPPED" {
result.Status = "SUCCESS"
}
newResults = append(newResults, result)
}
}
for _, variable := range oldExec.Workflow.WorkflowVariables {
workflowExecution.Workflow.WorkflowVariables = append(workflowExecution.Workflow.WorkflowVariables, variable)
}
for _, variable := range oldExec.Workflow.ExecutionVariables {
workflowExecution.Workflow.ExecutionVariables = append(workflowExecution.Workflow.ExecutionVariables, variable)
}
for _, variable := range oldExec.ExecutionVariables {
workflowExecution.ExecutionVariables = append(workflowExecution.ExecutionVariables, variable)
}
workflowExecution.Results = newResults
for _, result := range newResults {
workflowExecution.Workflow.FormControl.CleanupActions = append(workflowExecution.Workflow.FormControl.CleanupActions, result.Action.ID)
}
// Special handler for AI Agent things.
parentActionId := ""
if parentRequest != nil && parentRequest.URL != nil {
// Check for parameter "parent_node"
queries := parentRequest.URL.Query()
if queries != nil {
parentNode := queries.Get("parent_node")
if len(parentNode) > 0 {
parentActionId = parentNode
}
}
}
if len(parentActionId) > 0 {
// Makes them 'required' to run. Makes it possible to have conditions
// for AI Agents in workflows primarily
for _, branch := range oldExec.Workflow.Branches {
if branch.DestinationID != parentActionId {
continue
}
modifiedBranch := branch
modifiedBranch.DestinationID = action.ID
workflowExecution.Workflow.Branches = append(workflowExecution.Workflow.Branches, modifiedBranch)
}
}
workflowExecution.WorkflowId = action.SourceWorkflow
workflowExecution.Workflow.ID = action.SourceWorkflow
workflowExecution.ExecutionArgument = oldExec.ExecutionArgument
workflowExecution.ExecutionSource = action.SourceWorkflow
workflowExecution.ExecutionParent = action.SourceExecution
// Ensures it's set correctly
workflow.ID = action.SourceWorkflow
workflow.Actions = []Action{action}
workflowExecution.Workflow.Actions = []Action{action}
// Special handled for Decision reruns in AI Agents
// 1. Find the decision & reset cache
// 2. Update the execution itself to not have the relevant data
if len(decisionId) > 0 {
log.Printf("[DEBUG][%s] Handling Single action RERUN for AI Agent decision. DecisionID: %#v", oldExec.ExecutionId, decisionId)
if foundResultIndex == -1 {
return workflowExecution, errors.New("Failed to find the action. Please try again or contact support@shuffler.io if this persists.")
}
mappedOutput := AgentOutput{}
err = json.Unmarshal([]byte(oldExec.Results[foundResultIndex].Result), &mappedOutput)
if err != nil {
log.Printf("[ERROR][%s] Failed in decision output mapping (2): %s", oldExec.ExecutionId, err)
}
availableDecisions := []string{}
foundDecisionIndex := -1
decisionPosition := -1
newDecisions := []AgentDecision{}
for decisionIndex, decision := range mappedOutput.Decisions {
availableDecisions = append(availableDecisions, decision.RunDetails.Id)
if decision.RunDetails.Id != decisionId {
if decision.Action == "finish" {
if debug {
log.Printf("[DEBUG] Removing the 'finish' action due to rerun")
}
continue
}
newDecisions = append(newDecisions, decision)
continue
}
// The position in the hierarchy
decisionPosition = mappedOutput.Decisions[decisionIndex].I
foundDecisionIndex = decisionIndex
mappedOutput.CompletedAt = 0
mappedOutput.Decisions[decisionIndex].RunDetails.Status = "RUNNING"
mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt = time.Now().UnixMilli()
mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = 0
mappedOutput.Decisions[decisionIndex].RunDetails.RawResponse = ""
mappedOutput.Decisions[decisionIndex].RunDetails.DebugUrl = ""
newDecisions = append(newDecisions, mappedOutput.Decisions[decisionIndex])
}
if foundDecisionIndex == -1 {
return workflowExecution, errors.New(fmt.Sprintf("Failed to find and rerun decision '%s' out of '%s' in execution %s. Please try again or contact support@shuffler.io if the error persists.", decisionId, strings.Join(availableDecisions, ","), oldExec.ExecutionId))
}
// Removing everything AFTER the one we are currently on
// Has to be done in a wonky way due to not having ordered arrays
newNewDecisions := []AgentDecision{}
for _, newDecision := range newDecisions {
if decisionPosition != -1 && newDecision.I > decisionPosition && newDecision.RunDetails.Id != decisionId {
if debug {
log.Printf("[DEBUG] SKIPPING decision %s as it's after the rerun position", newDecision.RunDetails.Id)
}
continue
}
if newDecision.RunDetails.Status == "" {
continue
}
newNewDecisions = append(newNewDecisions, newDecision)
}
for newDecisionIndex, newDecision := range newNewDecisions {
if newDecision.RunDetails.Id == decisionId {
foundDecisionIndex = newDecisionIndex
}
}
mappedOutput.Decisions = newNewDecisions
mappedOutput.Status = "WAITING"
marshalledResult, err := json.Marshal(mappedOutput)
if err == nil {
oldExec.Results[foundResultIndex].Result = string(marshalledResult)
} else {
return workflowExecution, errors.New(fmt.Sprintf("Failed to marshal and rerun the decision. Please try again or contact support@shuffler.io if the error persists."))
}
oldExec.Results[foundResultIndex].Status = "WAITING"
oldExec.Results[foundResultIndex].CompletedAt = 0
oldExec.Results[foundResultIndex].Result = string(marshalledResult)
// Resets the action cache to ensure reruns happen
// 1. Update db & cache etc.
// 2. Force rerun the decision
oldExec.CompletedAt = 0
oldExec.Status = "EXECUTING"
// Action reset (in the workflow)
SetCache(ctx, fmt.Sprintf("%s_%s_result", oldExec.ExecutionId, oldExec.Results[foundResultIndex].Action.ID), marshalledResult, 60)
// Decision reset
DeleteCache(ctx, fmt.Sprintf("agent-%s-%s", oldExec.ExecutionId, decisionId))
// Decision run reset
go DeleteCache(ctx, fmt.Sprintf("agent_request_%s_%s_FINISHED", oldExec.ExecutionId, oldExec.Results[foundResultIndex].Action.ID))
go DeleteCache(ctx, fmt.Sprintf("agent_request_%s_%s_SUCCESS", oldExec.ExecutionId, oldExec.Results[foundResultIndex].Action.ID))
go DeleteCache(ctx, fmt.Sprintf("agent_request_%s_%s_ABORTED", oldExec.ExecutionId, oldExec.Results[foundResultIndex].Action.ID))
go DeleteCache(ctx, fmt.Sprintf("agent_request_%s_%s_FAILURE", oldExec.ExecutionId, oldExec.Results[foundResultIndex].Action.ID))
// Execution reset
executionCacheKey := fmt.Sprintf("workflowexecution_%s", oldExec.ExecutionId)
DeleteCache(ctx, executionCacheKey)
marshalledTotalResult, err := json.Marshal(oldExec)
if err == nil {
SetCache(ctx, executionCacheKey, marshalledTotalResult, 30)
}
SetWorkflowExecution(ctx, *oldExec, true)
go RunAgentDecisionAction(*oldExec, mappedOutput, mappedOutput.Decisions[foundDecisionIndex])
// FIXME: This is to ensure hadnling of the EXACT SAME decision happens.
return workflowExecution, errors.New(fmt.Sprintf("Successfully started rerun of decision %s. This will replace the current result.", decisionId))
}
}
// Overwriting, as the user should be in control
if len(originalUrl) > 0 && len(workflowExecution.Workflow.Actions) > 0 && !shuffleAuthInjected && len(action.AuthenticationId) == 0 {
for paramIndex, param := range workflowExecution.Workflow.Actions[0].Parameters {
if param.Name == "url" {
workflowExecution.Workflow.Actions[0].Parameters[paramIndex].Value = originalUrl
break
}
}
}
if user.ActiveOrg.Id != "" {
workflow.ExecutingOrg = user.ActiveOrg
workflowExecution.ExecutionOrg = user.ActiveOrg.Id
workflowExecution.OrgId = user.ActiveOrg.Id
}
if len(workflowExecution.ExecutionSource) == 0 || workflowExecution.ExecutionSource == "default" {
workflowExecution.ExecutionSource = "single_action"
}
if len(workflowExecution.Workflow.Name) == 0 {
workflowExecution.Workflow.Name = fmt.Sprintf("%s Single app run", action.AppName)
}
go SetWorkflowExecution(context.Background(), workflowExecution, true)
/*
err = SetWorkflowExecution(context.Background, workflowExecution, true)
if err != nil {
log.Printf("[WARNING] Failed handling single execution setup: %s", err)
return workflowExecution, err
}
*/
return workflowExecution, nil
}
// Handles the return of a single action
func HandleRetValidation(ctx context.Context, workflowExecution WorkflowExecution, resultAmount int, timeout int, actionId ...string) SingleResult {
findActionId := ""
if len(actionId) > 0 {
findActionId = actionId[0]
}
cnt := 0
returnBody := SingleResult{
Success: true,
Id: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: "",
Errors: []string{},
Validation: workflowExecution.Workflow.Validation,
// In case input parameters are wanted. This can happen due to translation.
Parameters: []WorkflowAppActionParameter{},
}
// VERY short sleeptime here on purpose
startTime := time.Now().Unix()
maxSeconds := 15
if project.Environment != "cloud" {
maxSeconds = 180
}
if timeout > maxSeconds {
maxSeconds = timeout
}
if debug {
log.Printf("[DEBUG][%s] Starting single action execution check. Max seconds: %d", workflowExecution.ExecutionId, maxSeconds)
}
addedParams := []string{}
sleeptime := 100
for {
time.Sleep(time.Duration(sleeptime) * time.Millisecond)
newExecution, err := GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
if err != nil {
// In case we are too fast
if cnt > 2 {
log.Printf("[WARNING][%s] Failed getting single execution data: %s", workflowExecution.ExecutionId, err)
break
}
}
returnBody.Validation = newExecution.Workflow.Validation
relevantIndex := -1
if len(findActionId) > 0 {
found := false
for i, res := range newExecution.Results {
if res.Action.ID == findActionId {
relevantIndex = i
found = true
break
}
}
if !found {
continue
}
}
//log.Printf("\n\n\n[INFO] Checking single action execution %s. Status: %s. Len: %d, resultAmount: %d", workflowExecution.ExecutionId, newExecution.Status, len(newExecution.Results), resultAmount-1)
if len(newExecution.Results) > resultAmount-1 {
if relevantIndex == -1 {
relevantIndex = len(newExecution.Results) - 1
}
if len(newExecution.Results[relevantIndex].Result) > 0 || newExecution.Results[relevantIndex].Status == "SUCCESS" {
returnBody.Result = newExecution.Results[relevantIndex].Result
if len(newExecution.Results[relevantIndex].Action.Parameters) > 0 {
for _, param := range newExecution.Results[relevantIndex].Action.Parameters {
// Remove auth just in case
if param.Configuration && param.Name != "url" {
continue
}
if (strings.Contains(param.Name, "liquid") || strings.Contains(param.Name, "warning") || strings.Contains(param.Name, "error")) && !ArrayContains(returnBody.Errors, param.Value) {
returnBody.Errors = append(returnBody.Errors, param.Value)
} else {
if !ArrayContains(addedParams, param.Name) {
returnBody.Parameters = append(returnBody.Parameters, param)
addedParams = append(addedParams, param.Name)
}
}
}
}
// FIXME: This is a custom fix for single action custom runs.
// Wait for validation to have ran
if newExecution.Workflow.Validation.ValidationRan {
// FIXME: Check the return here. If there is an issue with custom_action doesn't exist, we rebuild it in realtime
if strings.Contains(returnBody.Result, "custom_action doesn't exist") {
log.Printf("[INFO] Custom action doesn't exist for action %s", newExecution.Results[relevantIndex].Action.ID)
// FIXME:
// 1. Get the app itself
// 2. Find the owner
// 3. Rebuild as if we are the owner from their own org-id
// 4. Run the validation again
if len(newExecution.Results[relevantIndex].Action.AppID) == 0 {
for _, action := range newExecution.Workflow.Actions {
if action.ID == newExecution.Results[relevantIndex].Action.ID {
newExecution.Results[relevantIndex].Action.AppID = action.AppID
break
}
}
}
go runAppRebuildFromSingleAction(newExecution.Results[relevantIndex].Action.AppID)
}
break
}
}
}
cnt += 1
// Use startTime instead:
//if cnt == (maxSeconds * (maxSeconds * 100 / sleeptime)) {
if time.Now().Unix()-startTime > int64(maxSeconds) {
returnBody.Success = true
returnBody.Errors = []string{fmt.Sprintf("Polling timed out after %d seconds. Use the /api/v1/streams API with body `{\"execution_id\": \"%s\", \"authorization\": \"%s\"}` to get the latest results", maxSeconds, workflowExecution.ExecutionId, workflowExecution.Authorization)}
break
}
}
if debug {
log.Printf("[DEBUG][%s] Single action execution check finished. Result len: %d, Errors: %#v", workflowExecution.ExecutionId, len(returnBody.Result), returnBody.Errors)
}
if len(returnBody.Result) == 0 && len(returnBody.Errors) == 0 {
returnBody.Success = false
}
return returnBody
}
func runAppRebuildFromSingleAction(appId string) {
log.Printf("[INFO] Rebuilding app '%s' due to custom action not existing", appId)
if len(appId) == 0 {
return
}
ctx := context.Background()
app, err := GetApp(ctx, appId, User{}, false)
if err != nil {
log.Printf("[WARNING] Error getting app (execute SINGLE app action - 2): %s", appId)
return
}
if !app.Generated {
log.Printf("[INFO] App %s (%s) is not generated. Not rebuilding", app.Name, app.ID)
return
}
parsedApi, err := GetOpenApiDatastore(ctx, app.ID)
if err != nil {
log.Printf("[WARNING] Failed getting openapi data for app %s: %s", app.Name, err)
return
}
// Get the owner account
user, err := GetUser(ctx, app.Owner)
if err != nil {
log.Printf("[WARNING] Failed getting user %s for app %s: %s", app.Owner, app.Name, err)
return
}
log.Printf("[INFO] Rebuilding app %s (%s) due to custom action not existing. Impersonating owner for the request to ensure ownership stays equal: %s (%s)", app.Name, app.ID, user.Username, user.Id)
parsedSwagger := map[string]interface{}{}
err = json.Unmarshal([]byte(parsedApi.Body), &parsedSwagger)
if err != nil {
return
}
parsedSwagger["editing"] = true
parsedSwagger["id"] = app.ID
newSwagger, err := json.Marshal(parsedSwagger)
if err != nil {
log.Printf("[WARNING] Failed marshalling parsed swagger for app %s: %s", app.Name, err)
return
}
// Sending a localhost request, properly based on cloud/not cloud
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 && strings.Contains(os.Getenv("SHUFFLE_CLOUDRUN_URL"), "http") {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
if len(backendUrl) == 0 && project.Environment != "cloud" {
backendUrl = "http://localhost:5001"
}
requestDestination := fmt.Sprintf("%s/api/v1/verify_openapi", backendUrl)
request, err := http.NewRequest(
"POST",
requestDestination,
bytes.NewBuffer(newSwagger),
)
if err != nil {
log.Printf("[WARNING] Failed creating request for app %s: %s", app.Name, err)
return
}
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", user.ApiKey))
request.Header.Set("Org-Id", user.ActiveOrg.Id)
log.Printf("[INFO] Sending rebuild request to %s for app %s", requestDestination, app.Name)
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Printf("[WARNING] Failed sending request for app %s: %s", app.Name, err)
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("[WARNING] Failed reading response for app rebuild %s: %s", app.Name, err)
return
}
if resp.StatusCode != 200 && resp.StatusCode != 201 {
log.Printf("[WARNING] Failed rebuilding app %s: %s", app.Name, string(body))
return
}
log.Printf("[INFO] Successfully rebuilt app %s (%s): %s", app.Name, app.ID, string(body))
}
func GetDocs(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
location := strings.Split(request.URL.String(), "/")
if len(location) < 5 {
resp.WriteHeader(404)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`)))
return
}
if strings.Contains(location[4], "?") {
location[4] = strings.Split(location[4], "?")[0]
}
ctx := GetContext(request)
downloadLocation, downloadOk := request.URL.Query()["location"]
version, versionOk := request.URL.Query()["version"]
cacheKey := fmt.Sprintf("docs_%s", location[4])
if downloadOk {
cacheKey = fmt.Sprintf("%s_%s", cacheKey, downloadLocation[0])
}
if versionOk {
cacheKey = fmt.Sprintf("%s_%s", cacheKey, version[0])
}
// Look for 'folder' query
path := "docs"
folder, folderOk := request.URL.Query()["folder"]
if folderOk && len(folder) > 0 {
if strings.Contains(folder[0], "..") || strings.Contains(folder[0], "/") {
// Disallow traversal even if it's github
} else {
path = folder[0]
}
}
cacheKey = fmt.Sprintf("%s_%s", cacheKey, path)
resetCache := request.URL.Query().Get("resetCache") == "true"
if !resetCache {
cache, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(cache.([]uint8))
resp.WriteHeader(200)
resp.Write(cacheData)
return
}
}
owner := "shuffle"
repo := "shuffle-docs"
docPath := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/master/%s/%s.md", owner, repo, path, location[4])
// FIXME: User controlled and dangerous (possibly). Uses Markdown on the frontend to render it
realPath := ""
newname := location[4]
if downloadOk {
if downloadLocation[0] == "openapi" {
newname = strings.ReplaceAll(strings.ToLower(location[4]), `%20`, "_")
docPath = fmt.Sprintf("https://raw.githubusercontent.com/Shuffle/openapi-apps/master/docs/%s.md", newname)
realPath = fmt.Sprintf("https://github.com/Shuffle/openapi-apps/blob/master/docs/%s.md", newname)
} else if downloadLocation[0] == "python" && versionOk {
// Apparently this uses dashes for no good reason?
// Should maybe move everything over to underscores later?
newname = strings.ReplaceAll(newname, `%20`, "-")
newname = strings.ReplaceAll(newname, ` `, "-")
newname = strings.ReplaceAll(newname, `_`, "-")
newname = strings.ToLower(newname)
if version[0] == "1.0.0" {
docPath = fmt.Sprintf("https://raw.githubusercontent.com/Shuffle/python-apps/master/%s/1.0.0/README.md", newname)
realPath = fmt.Sprintf("https://github.com/Shuffle/python-apps/blob/master/%s/1.0.0/README.md", newname)
log.Printf("[INFO] Should download python app for version %s: %s", version[0], docPath)
} else {
realPath = fmt.Sprintf("https://github.com/Shuffle/python-apps/blob/master/%s/README.md", newname)
docPath = fmt.Sprintf("https://raw.githubusercontent.com/Shuffle/python-apps/master/%s/README.md", newname)
}
}
}
//log.Printf("Docpath: %s", docPath)
httpClient := &http.Client{}
req, err := http.NewRequest(
"GET",
docPath,
nil,
)
if err != nil {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`)))
resp.WriteHeader(404)
return
}
newresp, err := httpClient.Do(req)
if err != nil {
resp.WriteHeader(404)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`)))
return
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"}`)))
return
}
commitOptions := &github.CommitsListOptions{
Path: fmt.Sprintf("%s/%s.md", path, location[4]),
}
parsedLink := fmt.Sprintf("https://github.com/%s/%s/blob/master/%s/%s.md", owner, repo, path, location[4])
if len(realPath) > 0 {
parsedLink = realPath
}
token := os.Getenv("GITHUB_DOCS_READ_TOKEN")
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
tc := oauth2.NewClient(ctx, ts)
client := github.NewClient(tc)
githubResp := GithubResp{
Name: location[4],
Contributors: []GithubAuthor{},
Edited: "",
ReadTime: len(body) / 10 / 250,
Link: parsedLink,
}
if githubResp.ReadTime == 0 {
githubResp.ReadTime = 1
}
info, _, err := client.Repositories.ListCommits(ctx, owner, repo, commitOptions)
if err != nil {
log.Printf("[WARNING] Failed getting commit info: %s", err)
} else {
//log.Printf("Info: %s", info)
for _, commit := range info {
//log.Printf("Commit: %s", commit.Author)
newAuthor := GithubAuthor{}
if commit.Author != nil && commit.Author.AvatarURL != nil {
newAuthor.ImageUrl = *commit.Author.AvatarURL
}
if commit.Author != nil && commit.Author.HTMLURL != nil {
newAuthor.Url = *commit.Author.HTMLURL
}
found := false
for _, contributor := range githubResp.Contributors {
if contributor.Url == newAuthor.Url {
found = true
break
}
}
if !found && len(newAuthor.Url) > 0 && len(newAuthor.ImageUrl) > 0 {
githubResp.Contributors = append(githubResp.Contributors, newAuthor)
}
}
}
type Result struct {
Success bool `json:"success"`
Reason string `json:"reason"`
Meta GithubResp `json:"meta"`
}
var result Result
result.Success = true
result.Meta = githubResp
result.Reason = string(body)
b, err := json.Marshal(result)
if err != nil {
http.Error(resp, err.Error(), 500)
return
}
err = SetCache(ctx, cacheKey, b, 10080)
if err != nil {
log.Printf("[WARNING] Failed setting cache for doc %s: %s", location[4], err)
}
resp.WriteHeader(200)
resp.Write(b)
}
func GetDocList(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
ctx := GetContext(request)
path := "docs"
// Look for 'folder' query
folder, folderOk := request.URL.Query()["folder"]
if folderOk && len(folder) > 0 {
if strings.Contains(folder[0], "..") || strings.Contains(folder[0], "/") {
// Disallow traversal even if it's github
} else {
path = folder[0]
}
}
cacheKey := fmt.Sprintf("docs_list_%s", path)
result := FileList{}
resetCache := request.URL.Query().Get("resetCache") == "true"
if !resetCache {
cache, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(cache.([]uint8))
resp.WriteHeader(200)
resp.Write(cacheData)
return
}
}
token := os.Getenv("GITHUB_DOCS_READ_TOKEN")
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
tc := oauth2.NewClient(ctx, ts)
client := github.NewClient(tc)
owner := "shuffle"
repo := "shuffle-docs"
_, item1, _, err := client.Repositories.GetContents(ctx, owner, repo, path, nil)
if err != nil {
log.Printf("[WARNING] Failed getting docs list: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error listing directory"}`)))
return
}
if len(item1) == 0 {
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."}`)))
return
}
names := []GithubResp{}
for _, item := range item1 {
if !strings.HasSuffix(*item.Name, "md") {
continue
}
publishedDate := time.Now().Unix()
if path == "articles" {
commits, resp, err := client.Repositories.ListCommits(ctx, owner, repo, &github.CommitsListOptions{
Path: fmt.Sprintf("%s/%s", path, *item.Name),
})
if err != nil {
log.Printf("[WARNING] Failed getting commits for %s: %s", *item.Name, err)
if resp != nil {
log.Printf("[DEBUG] Response status: %d", resp.StatusCode)
}
} else {
if len(commits) > 0 {
publishedDate = commits[len(commits)-1].Commit.Author.Date.Unix()
}
}
}
// FIXME: Scuffed readtime calc
// Average word length = 5. Space = 1. 5+1 = 6 avg.
// Words = *item.Size/6/250
//250 = average read time / minute
// Doubling this for bloat removal in Markdown~
githubResp := GithubResp{
Name: (*item.Name)[0 : len(*item.Name)-3],
Contributors: []GithubAuthor{},
PublishedDate: publishedDate,
Edited: "",
ReadTime: *item.Size / 6 / 250,
Link: fmt.Sprintf("https://github.com/%s/%s/blob/master/%s/%s", owner, repo, path, *item.Name),
}
names = append(names, githubResp)
}
if path == "articles" {
// Sort articles by published date (newest first)
sort.Slice(names, func(i, j int) bool {
return names[i].PublishedDate > names[j].PublishedDate
})
}
//log.Printf(names)
result.Success = true
result.Reason = "Success"
result.List = names
b, err := json.Marshal(result)
if err != nil {
http.Error(resp, err.Error(), 500)
return
}
err = SetCache(ctx, cacheKey, b, 10080)
if err != nil {
log.Printf("[WARNING] Failed setting cache for cachekey %s: %s", cacheKey, err)
}
resp.WriteHeader(200)
resp.Write(b)
}
func md5sum(data []byte) string {
hasher := md5.New()
hasher.Write(data)
newmd5 := hex.EncodeToString(hasher.Sum(nil))
return newmd5
}
// Checks if data is sent from Worker >0.8.51, which sends a full execution
// instead of individial results
func ValidateNewWorkerExecution(ctx context.Context, body []byte, shouldReset bool) error {
var execution WorkflowExecution
err := json.Unmarshal(body, &execution)
if err != nil {
log.Printf("[WARNING] Failed execution unmarshaling: %s", err)
if strings.Contains(fmt.Sprintf("%s", err), "array into") {
parsedBody := string(body)
if len(parsedBody) > 500 {
parsedBody = parsedBody[0:500] + "..."
}
log.Printf("[ERROR] Array unmarshal error in validate new worker execution: %s", parsedBody)
}
return err
}
if len(execution.ExecutionId) == 0 {
//log.Printf("[ERROR] No execution id provided to validate new worker")
return errors.New("No execution id provided to validate new worker (2)")
}
baseExecution, err := GetWorkflowExecution(ctx, execution.ExecutionId)
if err != nil {
log.Printf("[ERROR][%s] Failed getting execution (workflowqueue): %s", execution.ExecutionId, err)
return err
}
if baseExecution.Authorization != execution.Authorization {
return errors.New("Bad authorization when validating execution")
}
// used to validate if it's actually the right marshal
if len(baseExecution.Workflow.Actions) != len(execution.Workflow.Actions) {
return errors.New(fmt.Sprintf("Bad length of actions (probably normal app): %d", len(execution.Workflow.Actions)))
}
if len(baseExecution.Workflow.Triggers) != len(execution.Workflow.Triggers) {
return errors.New(fmt.Sprintf("Bad length of trigger: %d (probably normal app)", len(execution.Workflow.Triggers)))
}
if baseExecution.Status == "FINISHED" || baseExecution.Status == "ABORTED" || baseExecution.Status == "FAILURE" {
log.Printf("[INFO][%s] Execution is already finished, not overriding", execution.ExecutionId)
return errors.New("Execution is already finished, not overriding")
}
if len(baseExecution.Results) > len(execution.Results) {
if shouldReset == true {
// Letting it pass and override. This is to ensure worker can override
log.Printf("[INFO][%s] Allowing workflow execution override with status %s, %d results and %d actions", execution.ExecutionId, execution.Status, len(execution.Results), len(execution.Workflow.Actions))
// Reset cache for all action results for Fixexecution
for _, result := range baseExecution.Results {
DeleteCache(ctx, fmt.Sprintf("%s_%s_result", execution.ExecutionId, result.Action.ID))
DeleteCache(ctx, fmt.Sprintf("%s_%s_sent", execution.ExecutionId, result.Action.ID))
}
} else {
return errors.New(fmt.Sprintf("Can't have less actions in a full execution than what exists: %d (old) vs %d (new)", len(baseExecution.Results), len(execution.Results)))
}
}
if execution.Status == "EXECUTING" {
//log.Printf("[INFO] Inside executing.")
extra := 0
for _, trigger := range execution.Workflow.Triggers {
//log.Printf("Appname trigger (0): %s", trigger.AppName)
if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
// 0x0elliot: remove log later
log.Printf("[INFO][%s] Found user input or shuffle workflow trigger. Adding to action count", execution.ExecutionId)
extra += 1
}
}
if len(execution.Workflow.Actions)+extra == len(execution.Results) {
// 0x0elliot: remove log later
log.Printf("[INFO][%s] All actions have results (%d). Setting status to FINISHED. Actions: %d, Triggers: %d, Results: %d", execution.ExecutionId, len(execution.Workflow.Actions)+extra, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results))
execution.Status = "FINISHED"
}
}
// Finds if subflow HAS a value when it should, otherwise it's not being set
for _, result := range execution.Results {
//log.Printf("%s = %s", result.Action.AppName, result.Status)
if result.Action.AppName != "shuffle-subflow" {
continue
}
if result.Status == "SKIPPED" {
continue
}
for _, trigger := range baseExecution.Workflow.Triggers {
if trigger.ID != result.Action.ID {
continue
}
for _, param := range trigger.Parameters {
if param.Name == "check_result" && param.Value == "true" {
//log.Printf("Found check as true!")
var subflowData SubflowData
err = json.Unmarshal([]byte(result.Result), &subflowData)
if err != nil {
log.Printf("Failed unmarshal in subflow check for %s: %s", result.Result, err)
} else if len(subflowData.Result) == 0 {
log.Printf("There is no result yet. Don't save?")
} else {
//log.Printf("There is a result: %s", result.Result)
}
break
}
}
break
}
}
// Check status is finished, and set timestamp for finished if it's 0
if execution.Status == "FINISHED" || execution.Status == "ABORTED" || execution.Status == "FAILURE" {
if baseExecution.CompletedAt == 0 {
baseExecution.CompletedAt = time.Now().Unix()
}
}
err = SetWorkflowExecution(ctx, execution, true)
executionSet := true
if err == nil {
log.Printf("[INFO][%s] Set workflowexecution based on new worker (>0.8.53) for workflow %s. Actions: %d, Triggers: %d, Results: %d, Status: %s", execution.ExecutionId, execution.WorkflowId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status)
executionSet = true
if execution.Status == "FINISHED" || execution.Status == "ABORTED" {
//log.Printf("[INFO][%s] Execution is finished or aborted. Incrementing cache statistics", execution.ExecutionId)
HandleExecutionCacheIncrement(ctx, execution)
}
} else {
log.Printf("[WARNING] Failed setting the execution for new worker (>0.8.53) - retrying once: %s. ExecutionId: %s, Actions: %d, Triggers: %d, Results: %d, Status: %s", err, execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status)
// Retrying
time.Sleep(5 * time.Second)
err = SetWorkflowExecution(ctx, execution, true)
if err != nil {
log.Printf("[ERROR] Failed setting the execution for new worker (>0.8.53) - 2nd attempt: %s. ExecutionId: %s, Actions: %d, Triggers: %d, Results: %d, Status: %s", err, execution.ExecutionId, len(execution.Workflow.Actions), len(execution.Workflow.Triggers), len(execution.Results), execution.Status)
} else {
executionSet = true
}
}
// Long convoluted way of validating and setting the value of a subflow that is also a loop
// timing issues / non-queues
_ = executionSet
//if executionSet {
// RunFixParentWorkflowResult(ctx, execution)
//}
DeleteCache(ctx, fmt.Sprintf("workflowexecution_%s", execution.WorkflowId))
DeleteCache(ctx, fmt.Sprintf("workflowexecution_%s_50", execution.WorkflowId))
DeleteCache(ctx, fmt.Sprintf("workflowexecution_%s_100", execution.WorkflowId))
return nil
}
// Only returning error as the point is for the current workflow to update the parent workflow
func RunFixParentWorkflowResult(ctx context.Context, execution WorkflowExecution) error {
//log.Printf("IS IT SUBFLOW?")
if len(execution.ExecutionParent) > 0 && execution.Status != "EXECUTING" && (project.Environment == "onprem" || project.Environment == "cloud") {
parentExecution, err := GetWorkflowExecution(ctx, execution.ExecutionParent)
if err == nil {
//log.Printf("[DEBUG] Got parent execution: %s", parentExecution.ExecutionId)
isLooping := false
setExecution := true
shouldSetValue := false
for _, action := range parentExecution.Workflow.Actions {
if action.AppName == "User Input" || action.AppName == "Shuffle Workflow" || action.AppName == "shuffle-subflow" {
parentExecution.Workflow.Triggers = append(parentExecution.Workflow.Triggers, Trigger{
AppName: action.AppName,
Parameters: action.Parameters,
ID: action.ID,
})
}
}
for _, trigger := range parentExecution.Workflow.Triggers {
if trigger.ID != execution.ExecutionSourceNode {
continue
}
for _, param := range trigger.Parameters {
if param.Name == "workflow" && param.Value != execution.Workflow.ID {
setExecution = false
}
if param.Name == "argument" && strings.Contains(param.Value, "$") && strings.Contains(param.Value, ".#") {
isLooping = true
}
if param.Name == "check_result" && param.Value == "true" {
shouldSetValue = true
}
}
break
}
if !isLooping && setExecution && shouldSetValue && parentExecution.Status == "EXECUTING" {
log.Printf("[DEBUG] Its NOT looping. Should we override the value?")
return nil
} else if isLooping && setExecution && shouldSetValue && parentExecution.Status == "EXECUTING" {
log.Printf("[DEBUG] Parentexecutions' subflow IS looping and is correct workflow. Should find correct answer in the node's result. Length of results: %d", len(parentExecution.Results))
// 1. Find the action's existing result
// 2. ONLY update it if the action status is WAITING and workflow status is EXECUTING
// 3. IF all parts of the subflow execution are finished, set it to FINISHED
// 4. If result length == length of actions + extra, set it to FINISHED
// 5. Before setting parent execution, make sure to grab the latest version of the workflow again, in case processing time is slow
resultIndex := -1
updateIndex := -1
for parentResultIndex, result := range parentExecution.Results {
if result.Action.ID != execution.ExecutionSourceNode {
continue
}
//log.Printf("[DEBUG] Found action %s' results: %s", result.Action.ID, result.Result)
if result.Status != "WAITING" {
break
}
//result.Result
var subflowDataLoop []SubflowData
err = json.Unmarshal([]byte(result.Result), &subflowDataLoop)
if err != nil {
log.Printf("[DEBUG] Failed unmarshaling in set parent data: %s", err)
break
}
for subflowIndex, subflowResult := range subflowDataLoop {
if subflowResult.ExecutionId != execution.ExecutionId {
// Should look for the subflowresult for this one too
continue
}
//log.Printf("[DEBUG] Found right execution on index %d. Result: %s", subflowIndex, subflowResult.Result)
if len(subflowResult.Result) == 0 {
updateIndex = subflowIndex
}
resultIndex = parentResultIndex
break
}
}
// FIXME: MAY cause transaction issues.
if updateIndex >= 0 && resultIndex >= 0 {
//log.Printf("\n\n\n[DEBUG] Should update index %d in resultIndex %d with new result %s\n\n\n", updateIndex, resultIndex, execution.Result)
// Again, get the result, just in case, and update that exact value instantly
newParentExecution, err := GetWorkflowExecution(ctx, execution.ExecutionParent)
if err == nil {
var subflowDataLoop []SubflowData
err = json.Unmarshal([]byte(newParentExecution.Results[resultIndex].Result), &subflowDataLoop)
if err == nil {
subflowDataLoop[updateIndex].Result = execution.Result
subflowDataLoop[updateIndex].ResultSet = true
marshalledSubflow, err := json.Marshal(subflowDataLoop)
if err == nil {
newParentExecution.Results[resultIndex].Result = string(marshalledSubflow)
err = SetWorkflowExecution(ctx, *newParentExecution, true)
if err != nil {
log.Printf("[WARNING] Error saving parent execution in subflow setting: %s", err)
} else {
log.Printf("[DEBUG] Updated index %d in subflow result %d with value of length %d. IDS HAVE TO MATCH: %s vs %s", updateIndex, resultIndex, len(execution.Result), subflowDataLoop[updateIndex].ExecutionId, execution.ExecutionId)
}
}
// Validating if all are done and setting back to executing
allFinished := true
for _, parentResult := range newParentExecution.Results {
if parentResult.Action.ID != execution.ExecutionSourceNode {
continue
}
var subflowDataLoop []SubflowData
err = json.Unmarshal([]byte(parentResult.Result), &subflowDataLoop)
if err == nil {
for _, subflowResult := range subflowDataLoop {
if subflowResult.ResultSet != true {
allFinished = false
break
}
}
break
} else {
allFinished = false
break
}
}
// FIXME: This will break if subflow with loop is last node in two workflows in a row (main workflow -> []subflow -> []subflow)
// Should it send the whole thing back as a result to itself to be handled manually? :thinking:
if allFinished {
//newParentExecution.Results[resultIndex].Status = "SUCCESS"
extra := 0
for _, trigger := range newParentExecution.Workflow.Triggers {
//log.Printf("Appname trigger (0): %s", trigger.AppName)
if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
extra += 1
}
}
if len(newParentExecution.Workflow.Actions)+extra == len(newParentExecution.Results) {
newParentExecution.Status = "FINISHED"
}
err = SetWorkflowExecution(ctx, *newParentExecution, true)
if err != nil {
log.Printf("[ERROR] Failed updating setExecution to FINISHED and SUCCESS: %s", err)
}
}
} else {
log.Printf("[WARNING] Failed to unmarshal result in set parent subflow: %s", err)
}
//= newValue
} else {
log.Printf("[WARNING] Failed to update parent, because execution %s couldn't be found: %s", execution.ExecutionParent, err)
}
}
}
}
}
return nil
}
// Function to hash the long sso certificates and use it in queries
func ssoCertHash(normalized string) string {
sum := md5.Sum([]byte(normalized))
return hex.EncodeToString(sum[:])
}
func fixCertificate(parsedX509Key string) string {
parsedX509Key = strings.Replace(parsedX509Key, "
", "", -1)
if strings.Contains(parsedX509Key, "BEGIN CERT") && strings.Contains(parsedX509Key, "END CERT") {
parsedX509Key = strings.Replace(parsedX509Key, "-----BEGIN CERTIFICATE-----\n", "", -1)
parsedX509Key = strings.Replace(parsedX509Key, "-----BEGIN CERTIFICATE-----", "", -1)
parsedX509Key = strings.Replace(parsedX509Key, "-----END CERTIFICATE-----\n", "", -1)
parsedX509Key = strings.Replace(parsedX509Key, "-----END CERTIFICATE-----", "", -1)
}
// PingOne issue
parsedX509Key = strings.Replace(parsedX509Key, "\r\n", "", -1)
parsedX509Key = strings.Replace(parsedX509Key, "\n", "", -1)
parsedX509Key = strings.Replace(parsedX509Key, "\r", "", -1)
parsedX509Key = strings.Replace(parsedX509Key, " ", "", -1)
parsedX509Key = strings.TrimSpace(parsedX509Key)
//log.Printf("Len: %d", len(parsedX509Key))
//log.Printf("%s", parsedX509Key)
return parsedX509Key
}
// Example implementation of SSO, including a redirect for the user etc
// Should make this stuff only possible after login
func HandleOpenId(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
//https://dev-18062475.okta.com/oauth2/default/v1/authorize?client_id=oa3romteykJ2aMgx5d7&response_type=code&scope=openid&redirect_uri=http%3A%2F%2Flocalhost%3A5002%2Fapi%2Fv1%2Flogin_openid&state=state-296bc9a0-a2a2-4a57-be1a-d0e2fd9bb601&code_challenge_method=S256&code_challenge=codechallenge
// http://localhost:5001/api/v1/login_openid#id_token=asdasd&session_state=asde9d78d8-6535-45fe-848d-0efa9f119595
//code -> Token
ctx := GetContext(request)
skipValidation := false
openidUser := OpenidUserinfo{}
org := &Org{}
code := request.URL.Query().Get("code")
if len(code) == 0 {
// Check id_token grant info
if request.Method == "POST" {
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No code or id_token specified - body read error in POST"}`)))
resp.WriteHeader(401)
return
}
stateSplit := strings.Split(string(body), "&")
for _, innerstate := range stateSplit {
itemsplit := strings.Split(innerstate, "=")
if len(itemsplit) <= 1 {
log.Printf("[WARNING] No key:value: %s", innerstate)
continue
}
if itemsplit[0] == "id_token" {
token, err := VerifyIdToken(ctx, itemsplit[1])
if err != nil {
log.Printf("[ERROR] Bad ID token provided: %s", err)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad ID token provided"}`)))
resp.WriteHeader(401)
return
}
openidUser.Sub = token.Sub
openidUser.Email = token.Email
openidUser.Roles = token.Roles
org = &token.Org
skipValidation = true
break
}
}
}
if !skipValidation {
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No code specified"}`)))
resp.WriteHeader(401)
return
}
}
if !skipValidation {
state := request.URL.Query().Get("state")
if len(state) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No state specified"}`)))
return
}
stateBase, err := base64.StdEncoding.DecodeString(state)
if err != nil {
log.Printf("[ERROR] Failed base64 decode OpenID state: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed base64 decoding of state"}`)))
return
}
log.Printf("State: %s", stateBase)
foundOrg := ""
foundRedir := ""
foundChallenge := ""
stateSplit := strings.Split(string(stateBase), "&")
for _, innerstate := range stateSplit {
itemsplit := strings.Split(innerstate, "=")
//log.Printf("Itemsplit: %s", itemsplit)
if len(itemsplit) <= 1 {
log.Printf("[WARNING] No key:value: %s", innerstate)
continue
}
if itemsplit[0] == "org" {
foundOrg = strings.TrimSpace(itemsplit[1])
}
if itemsplit[0] == "redirect" {
foundRedir = strings.TrimSpace(itemsplit[1])
}
if itemsplit[0] == "challenge" {
foundChallenge = strings.TrimSpace(itemsplit[1])
}
}
//log.Printf("Challenge len2: %d", len(foundChallenge))
if len(foundOrg) == 0 {
log.Printf("[ERROR] No org specified in state")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No org specified in state"}`)))
return
}
org, err = GetOrg(ctx, foundOrg)
if err != nil {
log.Printf("[WARNING] Error getting org in OpenID: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Couldn't find the org for sign-in in Shuffle"}`))
return
}
clientId := org.SSOConfig.OpenIdClientId
tokenUrl := org.SSOConfig.OpenIdToken
if len(tokenUrl) == 0 {
log.Printf("[ERROR] No token URL specified for OpenID. OrgID: %s", foundOrg)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No token URL specified in org %s. Please make sure to specify a token URL in the /admin panel in Shuffle for OpenID Connect"}`, foundOrg)))
return
}
//log.Printf("Challenge: %s", foundChallenge)
body, err := RunOpenidLogin(ctx, clientId, tokenUrl, foundRedir, code, foundChallenge, org.SSOConfig.OpenIdClientSecret)
if err != nil {
log.Printf("[WARNING] Error with body read of OpenID Connect: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
openid := OpenidResp{}
err = json.Unmarshal(body, &openid)
if err != nil {
log.Printf("[WARNING] Error in Openid marshal: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Automated replacement
userInfoUrlSplit := strings.Split(org.SSOConfig.OpenIdAuthorization, "/")
userinfoEndpoint := strings.Join(userInfoUrlSplit[0:len(userInfoUrlSplit)-1], "/") + "/userinfo"
//userinfoEndpoint := strings.Replace(org.SSOConfig.OpenIdAuthorization, "/authorize", "/userinfo", -1)
log.Printf("Userinfo endpoint: %s", userinfoEndpoint)
client := &http.Client{}
req, err := http.NewRequest(
"GET",
userinfoEndpoint,
nil,
)
//req.Header.Add("accept", "application/json")
//req.Header.Add("cache-control", "no-cache")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", openid.AccessToken))
res, err := client.Do(req)
if err != nil {
log.Printf("[WARNING] OpenID client DO (2): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed userinfo request"}`))
return
}
defer res.Body.Close()
body, err = ioutil.ReadAll(res.Body)
if err != nil {
log.Printf("[WARNING] OpenID client Body (2): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed userinfo body parsing"}`))
return
}
err = json.Unmarshal(body, &openidUser)
if err != nil {
log.Printf("[WARNING] Error in Openid marshal (2): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
if len(openidUser.Sub) == 0 && len(openidUser.Email) == 0 {
log.Printf("[WARNING] No user found in openid login (2)")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// if project.Environment == "cloud" {
// log.Printf("[WARNING] Openid SSO is not implemented for cloud yet. User %s", openidUser.Sub)
// resp.WriteHeader(401)
// resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Cloud Openid is not available yet"}`)))
// return
// }
userName := strings.ToLower(strings.TrimSpace(openidUser.Email))
if !strings.Contains(userName, "@") {
log.Printf("[ERROR] Bad username, but allowing due to OpenID: %s. Full Subject: %#v", userName, openidUser)
}
if strings.Contains(userName, "@shuffler.io") {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Disabled for support users"}`))
return
}
redirectUrl := "https://shuffler.io/workflows"
if project.Environment != "cloud" {
redirectUrl = "http://localhost:3001/workflows"
if len(os.Getenv("SSO_REDIRECT_URL")) > 0 {
baseUrl := os.Getenv("SSO_REDIRECT_URL")
// Check if URL contains /api/v1/login_openid and replace with /workflows
if strings.Contains(baseUrl, "/api/v1/login_openid") {
redirectUrl = strings.Replace(baseUrl, "/api/v1/login_openid", "/workflows", 1)
} else if !strings.HasSuffix(baseUrl, "/workflows") {
// If URL doesn't end with /workflows, append it
redirectUrl = fmt.Sprintf("%s/workflows", baseUrl)
} else {
redirectUrl = baseUrl
}
}
}
if len(userName) == 0 {
log.Printf("[ERROR] Username (%v) is empty in OpenID login for org: %v", userName, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Username is empty"}`))
return
}
if strings.Contains(strings.ToLower(userName), "shuffler.io") {
log.Printf("[ERROR] Username (%v) contains invalid domain in OpenID login for org: %v", userName, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Invalid username"}"`))
return
}
users, err := FindGeneratedUser(ctx, strings.ToLower(strings.TrimSpace(userName)))
if err == nil && len(users) > 0 {
for _, user := range users {
log.Printf("%s - %s", user.GeneratedUsername, userName)
if user.GeneratedUsername == userName {
foundOrgInUser := false
for _, userOrg := range user.Orgs {
if userOrg == org.Id {
foundOrgInUser = true
break
}
}
// check whether user is in org or not
foundUserInOrg := false
var usr User
for _, usr = range org.Users {
if usr.Id == user.Id {
foundUserInOrg = true
break
}
}
if (!foundOrgInUser || !foundUserInOrg) && org.SSOConfig.AutoProvision {
log.Printf("[WARNING] User %s (%s) is not in org %s (%s). Please contact the administrator - (1)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "User not found in the org. Autoprovisioning is disabled. Please contact the admin of the org to allow auto-provisioning of user."}`)))
return
} else if !foundOrgInUser || !foundUserInOrg {
log.Printf("[INFO] User %s (%s) is not in org %s (%s). Auto-provisioning is enabled. Adding user to org - (1)", user.Username, user.Id, org.Name, org.Id)
if !foundOrgInUser {
user.Orgs = append(user.Orgs, org.Id)
}
if !foundUserInOrg {
org.Users = append(org.Users, user)
}
} else {
log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login! - (1)", user.Username, user.Id, userName)
}
// check whether role is required for org
if org.SSOConfig.RoleRequired {
foundRole := false
for _, role := range openidUser.Roles {
// check whether role matches with shuffle-admin, shuffle-user or shuffle-org-reader
if role == "shuffle-admin" || role == "shuffle-user" || role == "shuffle-org-reader" {
foundRole = true
}
}
if !foundRole {
log.Printf("[WARNING] User %s (%s) role is missing in respone for org %s (%s). Please contact the administrator - (1)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Role detail is missing. Please contact the administrator of org."}`)))
return
}
}
role := user.Role
roleChange := false
if len(openidUser.Roles) > 0 {
for _, newRole := range openidUser.Roles {
if newRole == "shuffle-admin" {
role = "admin"
user.Role = "admin"
roleChange = true
break
}
if newRole == "shuffle-user" {
role = "user"
user.Role = "user"
roleChange = true
break
}
if newRole == "shuffle-org-reader" {
role = "org-reader"
user.Role = "org-reader"
roleChange = true
break
}
}
}
//log.Printf("SESSION: %s", user.Session)
user.ActiveOrg = OrgMini{
Name: org.Name,
Id: org.Id,
Role: role,
}
expiration := time.Now().Add(8 * time.Hour)
if len(user.Session) == 0 {
log.Printf("[INFO] User does NOT have session - creating - (1)")
sessionToken := uuid.NewV4().String()
newCookie := ConstructSessionCookie(sessionToken, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
err = SetSession(ctx, user, sessionToken)
if err != nil {
log.Printf("[WARNING] Error creating session for user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`)))
return
}
user.Session = sessionToken
} else {
log.Printf("[INFO] user have session resetting session and cookies for user: %v - (1)", userName)
sessionToken := user.Session
newCookie := ConstructSessionCookie(sessionToken, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
err = SetSession(ctx, user, sessionToken)
if err != nil {
log.Printf("[WARNING] Error creating session for user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`)))
return
}
}
user.LoginInfo = append(user.LoginInfo, LoginInfo{
IP: GetRequestIp(request),
Timestamp: time.Now().Unix(),
})
//Store users last session as new session so user don't have to go through sso again while changing org.
user.UsersLastSession = user.Session
err = SetUser(ctx, &user, false)
if err != nil {
log.Printf("[WARNING] Failed updating user when setting session: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed user update during session storage (2)"}`))
return
}
if roleChange {
// change user role in org if change
for i, usr := range org.Users {
if usr.Id == user.Id {
org.Users[i].Role = role
break
}
}
}
if !foundUserInOrg || roleChange {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org when setting user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed org update during user storage (2)"}`))
return
}
}
//redirectUrl = fmt.Sprintf("%s?source=SSO&id=%s", redirectUrl, session)
http.Redirect(resp, request, redirectUrl+"?type=sso_login", http.StatusSeeOther)
return
}
}
}
// Normal user. Checking because of backwards compatibility. Shouldn't break anything as we have unique names
users, err = FindUser(ctx, strings.ToLower(strings.TrimSpace(userName)))
if err == nil && len(users) > 0 {
for _, user := range users {
if user.Username == userName {
// Checking whether the user is in the org
foundOrgInUser := false
for _, userOrg := range user.Orgs {
if userOrg == org.Id {
foundOrgInUser = true
break
}
}
// check whether user is in org or not
foundUserInOrg := false
var usr User
for _, usr = range org.Users {
if usr.Id == user.Id {
foundUserInOrg = true
break
}
}
if (!foundOrgInUser || !foundUserInOrg) && org.SSOConfig.AutoProvision {
log.Printf("[WARNING] User %s (%s) is not in org %s (%s). Please contact the administrator - (2)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "User not found in the org. Autoprovisioning is disabled. Please contact the admin of the org to allow auto-provisioning of user."}`)))
return
} else if !foundOrgInUser || !foundUserInOrg {
log.Printf("[INFO] User %s (%s) is not in org %s (%s). Auto-provisioning is enabled. Adding user to org - (2)", user.Username, user.Id, org.Name, org.Id)
if !foundOrgInUser {
user.Orgs = append(user.Orgs, org.Id)
}
if !foundUserInOrg {
org.Users = append(org.Users, user)
}
} else {
log.Printf("[AUDIT] Found user %s (%s) which matches SSO info for %s. Redirecting to login!- (2)", user.Username, user.Id, userName)
}
//log.Printf("SESSION: %s", user.Session)
// check whether role is required for org
if org.SSOConfig.RoleRequired {
foundRole := false
for _, role := range openidUser.Roles {
// check whether role matches with shuffle-admin, shuffle-user or shuffle-org-reader
if role == "shuffle-admin" || role == "shuffle-user" || role == "shuffle-org-reader" {
foundRole = true
}
}
if !foundRole {
log.Printf("[WARNING] User %s (%s) role is missing in respone for org %s (%s). Please contact the administrator - (1)", user.Username, user.Id, org.Name, org.Id)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Role detail is missing. Please contact the administrator of org."}`)))
return
}
}
role := user.Role
roleChange := false
if len(openidUser.Roles) > 0 {
for _, newRole := range openidUser.Roles {
if newRole == "shuffle-admin" {
role = "admin"
user.Role = "admin"
roleChange = true
break
}
if newRole == "shuffle-user" {
role = "user"
user.Role = "user"
roleChange = true
break
}
if newRole == "shuffle-org-reader" {
role = "org-reader"
user.Role = "org-reader"
roleChange = true
break
}
}
}
user.ActiveOrg = OrgMini{
Name: org.Name,
Id: org.Id,
Role: role,
}
expiration := time.Now().Add(8 * time.Hour)
if len(user.Session) == 0 {
log.Printf("[INFO] User does NOT have session - creating - (2)")
sessionToken := uuid.NewV4().String()
newCookie := ConstructSessionCookie(sessionToken, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
err = SetSession(ctx, user, sessionToken)
if err != nil {
log.Printf("[WARNING] Error creating session for user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`)))
return
}
user.Session = sessionToken
} else {
log.Printf("[INFO] user have session resetting session and cookies for user: %v - (2)", userName)
sessionToken := user.Session
newCookie := ConstructSessionCookie(sessionToken, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
err = SetSession(ctx, user, sessionToken)
if err != nil {
log.Printf("[WARNING] Error creating session for user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`)))
return
}
}
user.LoginInfo = append(user.LoginInfo, LoginInfo{
IP: GetRequestIp(request),
Timestamp: time.Now().Unix(),
})
//Store users last session as new session so user don't have to go through sso again while changing org.
user.UsersLastSession = user.Session
err = SetUser(ctx, &user, false)
if err != nil {
log.Printf("[WARNING] Failed updating user when setting session: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed user update during session storage (2)"}`))
return
}
if roleChange {
// change user role in org if change
for i, usr := range org.Users {
if usr.Id == user.Id {
org.Users[i].Role = role
break
}
}
}
if !foundUserInOrg || roleChange {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org when setting session: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed org update during session storage (2)"}`))
return
}
}
//redirectUrl = fmt.Sprintf("%s?source=SSO&id=%s", redirectUrl, session)
http.Redirect(resp, request, redirectUrl+"?type=sso_login", http.StatusSeeOther)
return
}
}
}
/*
orgs, err := GetAllOrgs(ctx)
if err != nil {
log.Printf("[WARNING] Failed finding orgs during SSO setup: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting valid organizations"}`)))
return
}
foundOrg := Org{}
for _, org := range orgs {
if len(org.ManagerOrgs) == 0 {
foundOrg = org
break
}
}
*/
if len(org.Id) == 0 {
log.Printf("[WARNING] Failed finding a valid org (default) without suborgs during SSO setup")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed finding valid SSO auto org"}`)))
return
}
if org.SSOConfig.AutoProvision {
log.Printf("[INFO] Auto-provisioning user is not allow for org %s (%s) - can not add new user %s - (3)", org.Name, org.Id, userName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "User not found in the org. Autoprovisioning is disabled. Please contact the admin of the org to allow auto-provisioning of user."}`)))
return
}
if org.SSOConfig.RoleRequired {
foundRole := false
for _, role := range openidUser.Roles {
// check whether role matches with shuffle-admin, shuffle-user or shuffle-org-reader
if role == "shuffle-admin" || role == "shuffle-user" || role == "shuffle-org-reader" {
foundRole = true
}
}
if !foundRole {
log.Printf("[WARNING] Role is missing in respone for username %s. Please contact the administrator - (3)", userName)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Role detail is missing. Please contact the administrator of org."}`)))
return
}
}
// Assign default role as "user" for generated user, else assign the role from openid if available
// Change active org role and user.role to assign role
role := "user"
if len(openidUser.Roles) > 0 {
for _, newRole := range openidUser.Roles {
if newRole == "shuffle-admin" {
role = "admin"
break
}
if newRole == "shuffle-user" {
role = "user"
break
}
if newRole == "shuffle-org-reader" {
role = "org-reader"
break
}
}
}
log.Printf("[AUDIT] Adding user %s with role %s to org %s (%s) through single sign-on", userName, role, org.Name, org.Id)
newUser := new(User)
// Random password to ensure its not empty
newUser.Password = uuid.NewV4().String()
newUser.Username = userName
newUser.GeneratedUsername = userName
newUser.Verified = true
newUser.Active = true
newUser.CreationTime = time.Now().Unix()
newUser.Orgs = []string{org.Id}
newUser.LoginType = "OpenID"
newUser.Role = role
newUser.Session = uuid.NewV4().String()
newUser.ActiveOrg = OrgMini{
Name: org.Name,
Id: org.Id,
Role: role,
}
if project.Environment == "cloud" {
newUser.Regions = []string{"https://shuffler.io"}
}
verifyToken := uuid.NewV4()
ID := uuid.NewV4()
newUser.Id = ID.String()
newUser.VerificationToken = verifyToken.String()
expiration := time.Now().Add(8 * time.Hour)
//if len(user.Session) == 0 {
log.Printf("[INFO] User does NOT have session - creating")
sessionToken := uuid.NewV4().String()
newCookie := ConstructSessionCookie(sessionToken, expiration)
http.SetCookie(resp, newCookie)
newCookie.Name = "__session"
http.SetCookie(resp, newCookie)
err = SetSession(ctx, *newUser, sessionToken)
if err != nil {
log.Printf("[WARNING] Error creating session for user: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting session"}`)))
return
}
newUser.Session = sessionToken
if project.Environment == "cloud" && org.RegionUrl != "https://shuffler.io" {
newUser.Regions = append(newUser.Regions, org.RegionUrl)
}
//Store users last session as new session so user don't have to go through sso again while changing org.
newUser.UsersLastSession = sessionToken
err = SetUser(ctx, newUser, true)
if err != nil {
log.Printf("[WARNING] Failed setting new user in DB: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed updating the user"}`)))
return
}
http.Redirect(resp, request, redirectUrl+"?type=sso_login", http.StatusSeeOther)
return
}
func HandleDisconnectSSO(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting GET SUBORG request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
// Only allow POST requests for security
if request.Method != "POST" {
resp.WriteHeader(405)
resp.Write([]byte(`{"success": false, "reason": "Method not allowed"}`))
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in disconnect SSO: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Authentication failed"}`))
return
}
// Get org ID from URL path or request body
var orgId string
location := strings.Split(request.URL.String(), "/")
if len(location) >= 5 {
orgId = location[4] // /api/v1/disconnect_sso/{orgId}
}
if len(orgId) == 0 {
// Try to get from request body as fallback
body, err := ioutil.ReadAll(request.Body)
if err == nil {
var requestData struct {
OrgId string `json:"org_id"`
}
if json.Unmarshal(body, &requestData) == nil {
orgId = requestData.OrgId
}
}
}
if len(orgId) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Missing org_id parameter"}`))
return
}
// Verify user has access to this org
hasAccess := false
for _, userOrg := range user.Orgs {
if userOrg == orgId {
hasAccess = true
break
}
}
if !hasAccess {
log.Printf("[WARNING] User %s (%s) tried to disconnect SSO for org %s they don't have access to", user.Username, user.Id, orgId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Access denied to this organization"}`))
return
}
ctx := GetContext(request)
// Get the current SSO info for this org
ssoInfo, exists := user.GetSSOInfo(orgId)
if !exists || len(ssoInfo.Sub) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No SSO connection found for this organization"}`))
return
}
// Log the disconnection for audit purposes
log.Printf("[AUDIT] User %s (%s) is disconnecting SSO (sub: %s) from org %s", user.Username, user.Id, ssoInfo.Sub, orgId)
// Remove SSO info for this org by filtering the slice
newSSOInfos := []SSOInfo{}
for _, info := range user.SSOInfos {
if info.OrgID != orgId {
newSSOInfos = append(newSSOInfos, info)
}
}
user.SSOInfos = newSSOInfos
// Save the updated user
err = SetUser(ctx, &user, true)
if err != nil {
log.Printf("[ERROR] Failed to update user after SSO disconnection: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to disconnect SSO account"}`))
return
}
log.Printf("[INFO] Successfully disconnected SSO for user %s (%s) from org %s", user.Username, user.Id, orgId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "SSO account disconnected successfully"}`))
}
// Downloads documentation from Github to be placed in an app/workflow as markdown
// Caching no matter what, with no retries
func DownloadFromUrl(ctx context.Context, url string) ([]byte, error) {
cacheKey := fmt.Sprintf("docs_%s", url)
cache, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(cache.([]uint8))
return cacheData, nil
}
httpClient := &http.Client{}
req, err := http.NewRequest(
"GET",
url,
nil,
)
if err != nil {
SetCache(ctx, cacheKey, []byte{}, 30)
return []byte{}, err
}
newresp, err := httpClient.Do(req)
if err != nil {
return []byte{}, err
}
//log.Printf("URL %s, RESP: %d", url, newresp.StatusCode)
if newresp.StatusCode != 200 {
SetCache(ctx, cacheKey, []byte{}, 30)
return []byte{}, errors.New(fmt.Sprintf("No body to handle for %s. Status: %d", url, newresp.StatusCode))
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
SetCache(ctx, cacheKey, []byte{}, 30)
return []byte{}, err
}
//log.Printf("Documentation: %s", string(body))
if len(body) > 0 {
err = SetCache(ctx, cacheKey, body, 30)
if err != nil {
log.Printf("[WARNING] Failed setting cache for workflow/app doc %s: %s", url, err)
}
return body, nil
}
SetCache(ctx, cacheKey, []byte{}, 30)
return []byte{}, errors.New(fmt.Sprintf("No body to handle for %s", url))
}
// New execution with firestore
// The slow parts of it on FIRST request without cache:
// - Env loading (450ms)
// - Org Auth loading (500ms)
func PrepareWorkflowExecution(ctx context.Context, workflow Workflow, request *http.Request, maxExecutionDepth int64) (WorkflowExecution, ExecInfo, string, error) {
// Check the URL for the workflow ID itself
if request != nil { // && len(workflow.Actions) == 0 {
// Parse out the the workflow ID from the url
splitUrl := strings.Split(request.URL.Path, "/")
if len(splitUrl) == 6 {
foundId := splitUrl[4]
if len(foundId) == 36 {
if workflow.ID != foundId {
log.Printf("[DEBUG] Updating Workflow ID from '%s' to '%s'", workflow.ID, foundId)
workflow.ID = foundId
}
parentWorkflow, err := GetWorkflow(ctx, workflow.ID, true)
if err == nil && len(parentWorkflow.ID) == 36 && len(parentWorkflow.Actions) > 0 {
workflow = *parentWorkflow
}
}
}
}
// Try again if there is no request available? These are backups if we don't have the data
if len(workflow.ID) == 36 && len(workflow.Actions) == 0 {
parentWorkflow, err := GetWorkflow(ctx, workflow.ID, true)
if err != nil {
log.Printf("[WARNING] Failed getting workflow for execution: %s", err)
} else {
if len(parentWorkflow.ID) > 0 && len(parentWorkflow.Actions) > 0 {
workflow = *parentWorkflow
}
}
}
var workflowExecution WorkflowExecution
workflowBytes, err := json.Marshal(workflow)
if err != nil {
log.Printf("[WARNING] Failed workflow unmarshal in execution: %s", err)
return workflowExecution, ExecInfo{}, "", err
}
//log.Printf(workflow)
err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow)
if err != nil {
log.Printf("[WARNING] Failed prepare execution unmarshaling: %s", err)
return workflowExecution, ExecInfo{}, "Failed unmarshal during execution", err
}
if len(workflow.OrgId) > 0 {
workflowExecution.ExecutionOrg = workflow.OrgId
workflowExecution.OrgId = workflow.OrgId
}
if len(workflow.ExecutingOrg.Id) == 0 && len(workflow.OrgId) > 0 {
workflow.ExecutingOrg.Id = workflow.OrgId
}
makeNew := true
parentExecution := &WorkflowExecution{}
start, startok := request.URL.Query()["start"]
if request.Method == "POST" {
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[ERROR] Failed request POST read: %s", err)
return workflowExecution, ExecInfo{}, "Failed getting body", err
}
request.Body = io.NopCloser(bytes.NewBuffer(body))
// This one doesn't really matter.
//log.Printf("[INFO][%s] Running POST execution with body of length %d for workflow %s", workflowExecution.ExecutionId, len(string(body)), workflowExecution.Workflow.ID)
if len(body) >= 4 {
if body[0] == 34 && body[len(body)-1] == 34 {
body = body[1 : len(body)-1]
}
if body[0] == 34 && body[len(body)-1] == 34 {
body = body[1 : len(body)-1]
}
}
authgroupName, authgroupNameOk := request.URL.Query()["authgroup"]
if authgroupNameOk {
//log.Printf("\n\nAuthgroup: %s\n\n", authgroupName[0])
workflowExecution.Authgroup = authgroupName[0]
}
sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"]
if sourceAuthOk {
workflowExecution.ExecutionSourceAuth = sourceAuth[0]
} else {
//log.Printf("[DEBUG] Did NOT get source workflow auth")
}
sourceNode, sourceNodeOk := request.URL.Query()["source_node"]
if sourceNodeOk {
workflowExecution.ExecutionSourceNode = sourceNode[0]
} else {
//log.Printf("[DEBUG] Did NOT get source workflow node")
}
//workflowExecution.ExecutionSource = "default"
sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"]
if sourceWorkflowOk {
//log.Printf("Got source workflow %s", sourceWorkflow)
workflowExecution.ExecutionSource = sourceWorkflow[0]
} else {
//log.Printf("[DEBUG] Did NOT get source workflow (real). Not critical, as it can be overwritten with reference execution matching.")
}
sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"]
referenceExecution, referenceExecutionOk := request.URL.Query()["reference_execution"]
if referenceExecutionOk {
sourceExecutionOk = true
sourceExecution = referenceExecution
}
if sourceExecutionOk {
//log.Printf("[INFO] Got source execution%s", sourceExecution)
workflowExecution.ExecutionParent = sourceExecution[0]
// FIXME: Get the execution and check count
//workflowExecution.SubExecutionCount += 1
parentExecution, err = GetWorkflowExecution(ctx, workflowExecution.ExecutionParent)
if err == nil {
workflowExecution.SubExecutionCount = parentExecution.SubExecutionCount + 1
}
// Subflow are JUST lower than manual executions
if workflowExecution.Priority == 0 {
workflowExecution.Priority = 9
}
} else {
//log.Printf("Did NOT get source execution")
}
// Checks whether the subflow has been ran before based on parent execution ID + parent execution node ID (always unique)
// Used to deduplicate runs
if len(workflowExecution.ExecutionParent) > 0 && len(workflowExecution.ExecutionSourceNode) > 0 {
// Check if it should be looping:
// 1. Get workflowExecution.ExecutionParent's workflow
// 2. Find the ExecutionSourceNode
// 3. Check if the value of it is looping
var parentErr error
if len(parentExecution.ExecutionId) == 0 {
parentExecution, parentErr = GetWorkflowExecution(ctx, workflowExecution.ExecutionParent)
}
allowContinuation := false
if parentErr == nil {
found := false
for _, trigger := range parentExecution.Workflow.Triggers {
if trigger.ID != workflowExecution.ExecutionSourceNode {
continue
}
found = true
//$Get_Offenses.# -> Allow to run more
for _, param := range trigger.Parameters {
if param.Name == "argument" {
if strings.Contains(param.Value, "$") && strings.Contains(param.Value, ".#") {
allowContinuation = true
break
}
}
}
if allowContinuation {
break
}
}
if !found {
// Added from subflow trigger -> action translation
for _, action := range parentExecution.Workflow.Actions {
if action.ID != workflowExecution.ExecutionSourceNode {
continue
}
found = true
//$Get_Offenses.# -> Allow to run more
for _, param := range action.Parameters {
if param.Name == "argument" {
if strings.Contains(param.Value, "$") && strings.Contains(param.Value, ".#") {
allowContinuation = true
break
}
}
}
if allowContinuation {
break
}
}
}
}
// FIXME: Is execution ID missing?
if debug {
log.Printf("[INFO][%s]Is this a loop? %v", workflowExecution.ExecutionId, allowContinuation)
}
if allowContinuation == false {
newExecId := fmt.Sprintf("%s_%s_subflowcheck", workflowExecution.ExecutionParent, workflowExecution.ExecutionSourceNode)
cache, err := GetCache(ctx, newExecId)
if err == nil {
cacheData := []byte(cache.([]uint8))
newexec := WorkflowExecution{}
log.Printf("[WARNING][%s] Subflow exec %s already found - returning", workflowExecution.ExecutionId, newExecId)
// Returning to be used in worker
err = json.Unmarshal(cacheData, &newexec)
if err == nil {
return newexec, ExecInfo{}, fmt.Sprintf("Subflow for %s has already been executed", newExecId), errors.New(fmt.Sprintf("Subflow for %s has already been executed", newExecId))
}
return workflowExecution, ExecInfo{}, fmt.Sprintf("Subflow for %s has already been executed", newExecId), errors.New(fmt.Sprintf("Subflow for %s has already been executed", newExecId))
} else {
if debug {
log.Printf("[ERROR] Failed to find cache for %s %s %s", workflowExecution.ExecutionParent, workflowExecution.ExecutionId, workflowExecution.ExecutionSourceNode)
}
}
cacheData := []byte("1")
err = SetCache(ctx, newExecId, cacheData, 2)
if err != nil {
log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err)
} else {
}
}
}
if len(string(body)) < 75 && len(string(body)) > 1 {
if debug {
log.Printf("[DEBUG][%s] Body: %s", workflowExecution.ExecutionId, string(body))
}
} else {
// Here for debug purposes
//log.Printf("[DEBUG][%s] Body len: %d", workflowExecution.ExecutionId, len(string(body)))
}
var execution ExecutionRequest
err = json.Unmarshal(body, &execution)
if err != nil {
if len(string(body)) < 100 {
log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: '%s'. Err: %s", string(body), err)
} else {
log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err)
}
}
// Ensuring it works even if startpoint isn't defined
if execution.Start == "" && len(body) > 0 && len(execution.ExecutionSource) == 0 && len(execution.ExecutionArgument) == 0 {
// Check if "execution_argument" in body
execution.ExecutionArgument = string(body)
}
// FIXME - this should have "execution_argument" from executeWorkflow frontend
//log.Printf("EXEC: %s", execution)
if len(execution.ExecutionArgument) > 0 {
workflowExecution.ExecutionArgument = execution.ExecutionArgument
}
if len(execution.ExecutionSource) > 0 {
workflowExecution.ExecutionSource = execution.ExecutionSource
if workflowExecution.Priority == 0 {
workflowExecution.Priority = 5
}
}
//log.Printf("Execution data: %s", execution)
if len(execution.Start) == 36 && len(workflow.Actions) > 0 {
//log.Printf("[INFO][%s] Should start execution on node %s", execution.ExecutionId, execution.Start)
workflowExecution.Start = execution.Start
found := false
newStartnode := ""
for _, action := range workflow.Actions {
if action.ID == execution.Start {
found = true
break
}
if action.IsStartNode {
newStartnode = action.ID
}
}
if !found {
if len(newStartnode) > 0 {
execution.Start = newStartnode
} else {
log.Printf("[ERROR] Action %s was NOT found, and no other startnode found! Exiting execution.", execution.Start)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start))
}
}
} else if len(execution.Start) > 0 {
//return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start))
}
if len(execution.ExecutionId) == 36 {
workflowExecution.ExecutionId = execution.ExecutionId
} else {
sessionToken := uuid.NewV4()
workflowExecution.ExecutionId = sessionToken.String()
}
} else {
// Check for parameters of start and ExecutionId
// This is mostly used for user input trigger
answer, answerok := request.URL.Query()["answer"]
referenceId, referenceok := request.URL.Query()["reference_execution"]
authorization, authorizationok := request.URL.Query()["authorization"]
if answerok && referenceok && authorizationok {
// If answer is false, reference execution with result
log.Printf("[INFO] Should update reference execution and return, no need for further execution! exec ref: %s. Auth: %s", referenceId[0], authorization[0])
// Get the reference execution
oldExecution, err := GetWorkflowExecution(ctx, referenceId[0])
if err != nil {
log.Printf("[INFO][%s] Failed getting execution (execution) %s", referenceId[0], err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Failed getting execution ID '%s' because it doesn't exist (answer).", referenceId[0]), err
}
if oldExecution.Workflow.ID != workflow.ID {
log.Printf("[INFO] Wrong workflowid!")
return workflowExecution, ExecInfo{}, fmt.Sprintf("Bad workflow ID in get %s", referenceId), errors.New("Bad workflow ID")
}
if authorization[0] != oldExecution.Authorization {
log.Printf("[AUDIT][%s] Wrong authorization for execution during userinput! %s vs %s", referenceId[0], authorization[0], oldExecution.Authorization)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Bad authorization in get %s", referenceId), errors.New("Bad authorization key")
}
if len(start) == 0 {
// Just guessing~
for _, trigger := range workflow.Triggers {
if trigger.AppName == "User Input" {
start = []string{trigger.ID}
break
}
}
}
agentic := false
decisionId := ""
if len(start) == 0 && request != nil {
decisionIds, decisionIdOk := request.URL.Query()["decision_id"]
if decisionIdOk {
log.Printf("[INFO][%s] Got decisionId '%s' to find inside Agentic action", oldExecution.ExecutionId, decisionIds[0])
decisionId = decisionIds[0]
agentic = true
if len(workflow.Actions) == 1 {
start = append(start, workflow.Actions[0].ID)
oldExecution.Results[0].Status = "WAITING"
} else {
// Can loop for it
nodeIds, nodeIdsOk := request.URL.Query()["node_id"]
if len(nodeIds) > 0 && nodeIdsOk {
start = append(start, nodeIds[0])
} else {
log.Printf("[ERROR] No Agentic Start node found for workflow %s during workflow continuation. Pass in '&node_id={action.id}. Decision ID: %#v", workflow.ID, decisionId)
}
}
}
}
if len(start) == 0 {
log.Printf("[ERROR] No start node found for workflow %s during workflow continuation", workflow.ID)
return workflowExecution, ExecInfo{}, fmt.Sprintf("No start node found for workflow continuation %s. Pass in node_id={action.id} to bypass", workflow.ID), errors.New("No start node found for workflow continuation")
}
//log.Printf("Result len: %d", len(oldExecution.Results))
newResults := []ActionResult{}
foundresult := ActionResult{}
for resultIndex, result := range oldExecution.Results {
if result.Action.ID == start[0] {
if result.Status == "ABORTED" {
log.Printf("[INFO][%s] Found aborted result: %s (%s)", oldExecution.ExecutionId, result.Action.Label, result.Action.ID)
if oldExecution.Status != "ABORTED" {
log.Printf("[INFO] Aborting execution %s as it should have already been aborted in the past", oldExecution.ExecutionId)
oldExecution.Status = "ABORTED"
oldExecution.CompletedAt = time.Now().Unix()
SetWorkflowExecution(ctx, *oldExecution, true)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Execution %s was already aborted", oldExecution.ExecutionId), errors.New("Execution already aborted")
}
}
}
// Handles agentic run continues
// This is if shuffler.io/agents => questions are answered
if agentic {
log.Printf("[INFO][%s] Should fix the decision by injecting the values and continuing to the next step! :3", oldExecution.ExecutionId)
// 1. Find the result inside the result.Result -> AgentOutput
unmarshalledDecision := AgentOutput{}
err = json.Unmarshal([]byte(result.Result), &unmarshalledDecision)
if err != nil {
log.Printf("[ERROR][%s] Failed unmarshalling decision inside agentic workflow: %s", oldExecution.ExecutionId, err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Failed handling decision for agentic runs. Please contact support@shuffler.io. Details: %s", err), err
}
// Finds the local exec argument sent from User Input UI
execArg := request.URL.Query().Get("execution_argument")
if len(execArg) == 0 {
note := request.URL.Query().Get("note")
if len(note) > 0 {
execArg = note
}
}
cleanupFailures := false
fieldsChanged := false
for decisionIndex, decision := range unmarshalledDecision.Decisions {
if decision.RunDetails.Id != decisionId {
continue
}
findContinue := false
if decision.Action == "finish" {
findContinue = true
}
if debug {
log.Printf("[DEBUG][%s] Found decision '%s' inside the agentic workflow to update. Exec arg: %#v", workflowExecution.ExecutionId, decisionId, execArg)
}
mappedArgument := map[string]string{}
err = json.Unmarshal([]byte(execArg), &mappedArgument)
if err != nil {
log.Printf("[ERROR][%s] Failed unmarshalling execution argument during agentic decision handling: %s", execArg, err)
break
}
handledNumber := []string{}
for key, value := range mappedArgument {
// Handles special case for Continuing an existing Agent run by modifying the "finish" action
log.Printf("[DEBUG][%s] Handling key '%s' with value '%s' during agentic decision handling. findContinue: %#v", oldExecution.ExecutionId, key, value, findContinue)
if key == "approve" && decision.RunDetails.Status == "WAITING" && (decision.Category == "singul" || decision.Category == "standalone") {
log.Printf("[INFO][%s] Approving decision '%s' with value '%s' during agentic decision handling. This will finish the decision and the execution if it's a standalone decision.", oldExecution.ExecutionId, key, value)
if value == "true" {
unmarshalledDecision.Status = "RUNNING"
decision.RunDetails.Status = "RUNNING"
decision.Fields = append(decision.Fields, Valuereplace{
Key: "approve",
Value: fmt.Sprintf("Approved to continue at %s", time.Now().Format(time.RFC1123)),
})
fieldsChanged = true
cleanupFailures = true
} else if value == "false" {
decision.RunDetails.Status = "FINISHED"
decision.Fields = append(decision.Fields, Valuereplace{
Key: "approve",
Value: fmt.Sprintf("Approval DENIED at %d. Should stop the agent.", time.Now().Unix()),
})
fieldsChanged = true
cleanupFailures = true
} else {
log.Printf("[ERROR][%s] Invalid value for 'approve': %s", oldExecution.ExecutionId, value)
}
unmarshalledDecision.Decisions[decisionIndex] = decision
break
}
if findContinue {
// The only key we care about in this case
if key == "continue" {
// Overwrite everything
if workflowExecution.Status == "FINISHED" {
workflowExecution.Status = "EXECUTING"
}
unmarshalledDecision.Status = "RUNNING"
unmarshalledDecision.Output = ""
decision.Fields = []Valuereplace{
Valuereplace{
Key: "continue",
Value: "How do you want to continue?",
Answer: value,
},
}
decision.Action = "ask"
decision.Tool = "ask"
decision.Category = "standalone"
// Make the value max 50 bytes
decision.Reason = fmt.Sprintf("User Input: %.50s", value)
unmarshalledDecision.Decisions[decisionIndex] = decision
fieldsChanged = true
cleanupFailures = true
break
}
continue
}
if ArrayContains(handledNumber, key) {
continue
}
handledNumber = append(handledNumber, key)
foundNumberSplit := strings.Split(key, "_")
if len(foundNumberSplit) != 2 {
continue
}
fieldNumber, err := strconv.Atoi(foundNumberSplit[1])
if err != nil {
continue
}
// Both start counting from 0
if len(decision.Fields) <= fieldNumber {
continue
}
if debug {
log.Printf("[DEBUG] Mapping field %d to value %s", fieldNumber, value)
}
decision.Fields[fieldNumber].Answer = value
fieldsChanged = true
}
if fieldsChanged {
decision.RunDetails.Status = "FINISHED"
decision.RunDetails.CompletedAt = time.Now().Unix()
unmarshalledDecision.Decisions[decisionIndex] = decision
// Updates cache live
decisionId := fmt.Sprintf("agent-%s-%s", oldExecution.ExecutionId, decision.RunDetails.Id)
marshalledDecision, err := json.Marshal(decision)
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling decision during agentic decision handling: %s", oldExecution.ExecutionId, err)
} else {
SetCache(ctx, decisionId, marshalledDecision, 60)
}
}
break
}
// Cleans them up to be "IGNORED" instead
// This is a concept to allow for better UX
// Only runs if you "continue" running an agent run
if cleanupFailures {
for decisionIndex, _ := range unmarshalledDecision.Decisions {
decision := unmarshalledDecision.Decisions[decisionIndex]
if decision.RunDetails.Status != "FAILURE" {
continue
}
decision.RunDetails.Status = "IGNORED"
unmarshalledDecision.Decisions[decisionIndex] = decision
decisionId := fmt.Sprintf("agent-%s-%s", oldExecution.ExecutionId, decision.RunDetails.Id)
marshalledDecision, err := json.Marshal(decision)
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling decision during agentic decision handling: %s", oldExecution.ExecutionId, err)
} else {
SetCache(ctx, decisionId, marshalledDecision, 60)
}
fieldsChanged = true
}
}
// This is a weird if statement, due to it being used for
// multiple purposes around field change & decision id finding
// FIXME: Not sure if this is correct to catch bad decision IDs
//if !fieldsChanged && cleanupFailures {
if !fieldsChanged {
//if cleanupFailures {
return workflowExecution, ExecInfo{}, fmt.Sprintf("Could not find fields for decision '%s'. Did you fill them in?", decisionId), errors.New(fmt.Sprintf("Could not find fields decision '%s'. Did you fill them in?", decisionId))
}
newDecisionBytes, err := json.Marshal(unmarshalledDecision)
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling decision inside agentic workflow: %s", oldExecution.ExecutionId, err)
} else {
result.Result = string(newDecisionBytes)
}
actionCacheId := fmt.Sprintf("%s_%s_result", oldExecution.ExecutionId, result.Action.ID)
err = SetCache(ctx, actionCacheId, []byte(result.Result), 35)
if err != nil {
log.Printf("[ERROR] Failed setting cache for action result %s: %s", actionCacheId, err)
}
oldExecution.Results[resultIndex] = result
// FIXME: Can we force continue the agent from here? Or do we send another action inbetween?
result.Status = fmt.Sprintf("%s_%s", "FINISHED", decisionId)
newExec, _, err := handleAgentDecisionStreamResult(*oldExecution, result)
if err != nil {
log.Printf("[ERROR][%s] Failed handling agentic decision result: %s", oldExecution.ExecutionId, err)
}
return *newExec, ExecInfo{}, fmt.Sprintf("Agentic question handled (%s)", oldExecution.ExecutionId), errors.New("User Input: Agentic question action handled successfully!")
} else if result.Status == "WAITING" && !agentic {
log.Printf("[INFO][%s] Found relevant User Input result: %s (%s)", result.ExecutionId, result.Action.Label, result.Action.ID)
var userinputResp UserInputResponse
err = json.Unmarshal([]byte(result.Result), &userinputResp)
// Error here should just be warnings
if err != nil {
log.Printf("[WARNING][%s] Failed unmarshalling userinput (not critical): %s", result.ExecutionId, err)
}
//if err == nil {
userinputResp.ClickInfo.Clicked = true
userinputResp.ClickInfo.Time = time.Now().Unix()
userinputResp.ClickInfo.IP = GetRequestIp(request)
userinputResp.ClickInfo.Note = ""
// Set success based on answer
if len(answer) > 0 && answer[0] == "false" {
userinputResp.Success = false
userinputResp.Reason = "User declined the input"
} else {
userinputResp.Success = true
userinputResp.Reason = "User approved the input"
}
// Check if the "note" parameter exists in the request
execArg := request.URL.Query().Get("execution_argument")
if len(execArg) > 0 {
userinputResp.ClickInfo.Note = execArg
}
note := request.URL.Query().Get("note")
if len(note) > 0 {
userinputResp.ClickInfo.Note = note
}
// FIXME: Validate their input if they answered or not
foundTrigger := Trigger{}
for _, trigger := range workflow.Triggers {
if trigger.ID == result.Action.ID {
foundTrigger = trigger
break
}
}
questions := []string{}
dedupedQuestions := []string{}
actualQuestions := []InputQuestion{}
for _, param := range foundTrigger.Parameters {
if param.Name != "input_questions" {
continue
}
err = json.Unmarshal([]byte(param.Value), &questions)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling input questions in %s in workflow %s: %s", foundTrigger.ID, workflow.ID, err)
continue
}
for _, question := range questions {
question := strings.ToLower(strings.TrimSpace(question))
if ArrayContains(dedupedQuestions, question) {
continue
}
dedupedQuestions = append(dedupedQuestions, question)
for _, inputQ := range workflow.InputQuestions {
if strings.ToLower(strings.TrimSpace(inputQ.Name)) == question {
actualQuestions = append(actualQuestions, inputQ)
}
}
}
break
}
if len(dedupedQuestions) > 0 {
mappedAnswer := map[string]string{}
if len(userinputResp.ClickInfo.Note) > 0 {
err = json.Unmarshal([]byte(userinputResp.ClickInfo.Note), &mappedAnswer)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling userinput note: %s", err)
}
missingFields := []string{}
for _, actualQuestion := range actualQuestions {
if strings.Contains(actualQuestion.Value, ";") {
actualQuestion.Value = strings.Split(actualQuestion.Value, ";")[0]
}
/*
// FIXME: Required check here
if actualQuestion.Required == false {
continue
}
*/
found := false
for key, value := range mappedAnswer {
if strings.ToLower(strings.TrimSpace(actualQuestion.Value)) != strings.ToLower(strings.TrimSpace(key)) {
continue
}
if len(value) > 0 {
found = true
}
break
}
if !found {
missingFields = append(missingFields, actualQuestion.Value)
}
}
if len(missingFields) > 0 {
return *oldExecution, ExecInfo{}, "Answer all questions first.", errors.New(fmt.Sprintf("Answer all questions: %s", strings.Join(missingFields, ", ")))
}
}
}
user, err := HandleApiAuthentication(nil, request)
if err == nil && user.Username != "" {
userinputResp.ClickInfo.User = user.Username
}
b, err := json.Marshal(userinputResp)
if err != nil {
log.Printf("[ERROR] Failed marshalling userinput: %s", err)
} else {
result.Result = string(b)
}
result.CompletedAt = int64(time.Now().Unix()) * 1000
log.Printf("[INFO][%s] Setting result to %s. Answer: %#v", oldExecution.ExecutionId, result.Action.Label, answer)
sendSelfRequest := false
if answer[0] == "false" {
result.Status = "SUCCESS"
log.Printf("[INFO][%s] User Input '%s' (%s) answered FALSE - status SUCCESS, answer stored in click_info.", oldExecution.ExecutionId, result.Action.Label, result.Action.ID)
} else {
result.Status = "SUCCESS"
log.Printf("[INFO][%s] User Input '%s' (%s) answered TRUE - status SUCCESS.", oldExecution.ExecutionId, result.Action.Label, result.Action.ID)
}
// Should send result to self?
fullMarshal, err := json.Marshal(result)
log.Printf("[DEBUG] Result to send: %s", string(fullMarshal))
if err != nil {
log.Printf("[ERROR][%s] Failed marshalling userinput result: %s", oldExecution.ExecutionId, err)
} else {
actionCacheId := fmt.Sprintf("%s_%s_result", result.ExecutionId, result.Action.ID)
err = SetCache(ctx, actionCacheId, fullMarshal, 35)
if err != nil {
log.Printf("[ERROR] Failed setting cache for action result %s: %s", actionCacheId, err)
}
// Answer=false: save result, trigger failure subflow if configured, then finish.
if answer[0] == "false" {
log.Printf("[INFO][%s] User Input '%s' answered NO - saving result and finishing execution.", oldExecution.ExecutionId, result.Action.Label)
// Update result in execution before triggering subflow (auth needs non-FINISHED status)
for newresIndex, newres := range oldExecution.Results {
if newres.Action.ID == result.Action.ID {
oldExecution.Results[newresIndex] = result
break
}
}
// Trigger failure subflow if configured
failureSubflowId := ""
failureSubflowStartnode := ""
log.Printf("[DEBUG][%s] Looking for subflow_failure param in trigger %s. Workflow has %d triggers.", oldExecution.ExecutionId, result.Action.ID, len(workflow.Triggers))
for _, trigger := range workflow.Triggers {
if trigger.ID == result.Action.ID {
log.Printf("[DEBUG][%s] Found matching trigger '%s' with %d params", oldExecution.ExecutionId, trigger.Label, len(trigger.Parameters))
for _, param := range trigger.Parameters {
log.Printf("[DEBUG][%s] Trigger param: name=%s, value=%s", oldExecution.ExecutionId, param.Name, param.Value)
if param.Name == "subflow_failure" && len(param.Value) > 0 {
failureSubflowId = param.Value
}
if param.Name == "subflow_failure_startnode" && len(param.Value) > 0 {
failureSubflowStartnode = param.Value
}
}
break
}
}
log.Printf("[DEBUG][%s] Failure subflow lookup result: id='%s', startnode='%s'", oldExecution.ExecutionId, failureSubflowId, failureSubflowStartnode)
if len(failureSubflowId) > 0 {
log.Printf("[INFO][%s] Triggering failure subflow %s for declined User Input '%s'", oldExecution.ExecutionId, failureSubflowId, result.Action.Label)
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
} else if project.Environment == "cloud" && len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
// Execution argument with decline context
execArgMap := map[string]interface{}{
"success": false,
"reason": userinputResp.Reason,
"source_workflow": workflow.ID,
"source_execution": oldExecution.ExecutionId,
"source_node": result.Action.ID,
"information": userinputResp.Information,
"click_info": userinputResp.ClickInfo,
}
execArgBytes, _ := json.Marshal(execArgMap)
execArg := string(execArgBytes)
runUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?source_workflow=%s&source_execution=%s&source_auth=%s&source_node=%s&start=%s",
backendUrl, failureSubflowId,
workflow.ID,
oldExecution.ExecutionId,
oldExecution.Authorization,
result.Action.ID,
failureSubflowStartnode,
)
reqBody := fmt.Sprintf(`{"execution_argument": %s}`, strconv.Quote(execArg))
topClient := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
req, err := http.NewRequest("POST", runUrl, bytes.NewBuffer([]byte(reqBody)))
if err != nil {
log.Printf("[ERROR][%s] Failed creating failure subflow request: %s", oldExecution.ExecutionId, err)
} else {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", oldExecution.Authorization))
resp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR][%s] Failed triggering failure subflow %s: %s", oldExecution.ExecutionId, failureSubflowId, err)
} else {
defer resp.Body.Close()
respBody, _ := ioutil.ReadAll(resp.Body)
log.Printf("[INFO][%s] Failure subflow %s triggered (status: %d)", oldExecution.ExecutionId, failureSubflowId, resp.StatusCode)
// Parse response to get execution ID and update result
var subflowResp struct {
Success bool `json:"success"`
ExecutionID string `json:"execution_id"`
Authorization string `json:"authorization"`
}
if jsonErr := json.Unmarshal(respBody, &subflowResp); jsonErr == nil && len(subflowResp.ExecutionID) > 0 {
userinputResp.DeclineSubflow.Success = subflowResp.Success
userinputResp.DeclineSubflow.ExecutionID = subflowResp.ExecutionID
userinputResp.DeclineSubflow.WorkflowID = failureSubflowId
frontendUrl := backendUrl
if strings.Contains(frontendUrl, "appspot.com") || strings.Contains(frontendUrl, "run.app") {
frontendUrl = "https://shuffler.io"
}
userinputResp.DeclineSubflowURL = fmt.Sprintf("%s/workflows/%s?execution_id=%s", frontendUrl, failureSubflowId, subflowResp.ExecutionID)
// Update result with decline subflow info
updatedResult, marshalErr := json.Marshal(userinputResp)
if marshalErr == nil {
result.Result = string(updatedResult)
oldExecution.Result = result.Result
for newresIndex, newres := range oldExecution.Results {
if newres.Action.ID == result.Action.ID {
oldExecution.Results[newresIndex] = result
break
}
}
// Update result with decline subflow info
updatedResult, marshalErr := json.Marshal(userinputResp)
if marshalErr == nil {
result.Result = string(updatedResult)
for newresIndex, newres := range oldExecution.Results {
if newres.Action.ID == result.Action.ID {
oldExecution.Results[newresIndex] = result
break
}
}
}
}
log.Printf("[INFO][%s] Decline subflow execution: %s, URL: %s", oldExecution.ExecutionId, subflowResp.ExecutionID, userinputResp.DeclineSubflowURL)
}
}
}
}
// Finish execution
oldExecution.Status = "FINISHED"
oldExecution.Result = result.Result
oldExecution.CompletedAt = int64(time.Now().Unix())
oldExecution.LastNode = result.Action.ID
err = SetWorkflowExecution(ctx, *oldExecution, true)
if err != nil {
log.Printf("[ERROR][%s] Failed saving finished execution after answer=false: %s", oldExecution.ExecutionId, err)
}
return *oldExecution, ExecInfo{}, "", errors.New("User Input: Execution stopped by user (answer=false)")
}
// Answer=true: save result and re-add to queue
if sendSelfRequest == false && strings.ToLower(result.Action.Environment) != "cloud" {
log.Printf("[DEBUG][%s] SETTING user input result, and re-adding it to queue IF not in worker. Environment: %s", result.ExecutionId, result.Action.Environment)
if project.Environment == "worker" {
log.Printf("\n\n[DEBUG][%s] Worker user input restart. What do? Should we ever reach this point?\n\n", project.Environment)
} else {
updateMade := true
log.Printf("[DEBUG][%s] Re-adding user input execution to db & queue after re-setting result back", result.ExecutionId)
oldExecution.Status = "EXECUTING"
for newresIndex, newres := range oldExecution.Results {
if newres.Action.ID == result.Action.ID {
oldExecution.Results[newresIndex] = result
break
}
}
err = SetWorkflowExecution(ctx, *oldExecution, true)
if err != nil {
updateMade = false
log.Printf("[ERROR] Failed setting workflow execution actionresult in execution: %s", err)
}
// Should re-add to queue
executionRequest := ExecutionRequest{
ExecutionId: oldExecution.ExecutionId,
WorkflowId: oldExecution.Workflow.ID,
Authorization: oldExecution.Authorization,
Environments: []string{result.Action.Environment},
}
// Increase priority on User Input catch-ups
executionRequest.Priority = 11
parsedEnv := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(result.Action.Environment, " ", "-"), "_", "-")), oldExecution.ExecutionOrg)
if project.Environment != "cloud" {
parsedEnv = strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(result.Action.Environment, " ", "-"), "_", "-"))
}
err = SetWorkflowQueue(ctx, executionRequest, parsedEnv)
if err != nil {
updateMade = false
log.Printf("[ERROR] Failed re-adding User Input execution to db: %s", err)
}
if updateMade {
return *oldExecution, ExecInfo{}, "", errors.New("User Input: Execution continued by user (answer=true)")
}
}
}
if sendSelfRequest || strings.ToLower(result.Action.Environment) == "cloud" {
backendUrl := os.Getenv("BASE_URL")
if project.Environment != "cloud" {
port := 5001
if os.Getenv("BACKEND_PORT") != "" {
// Read it to int
newPort, err := strconv.Atoi(os.Getenv("BACKEND_PORT"))
if err != nil {
log.Printf("[ERROR] Failed converting BACKEND_PORT to int: %s", err)
} else {
port = newPort
}
}
// Selfrequest
backendUrl = fmt.Sprintf("http://localhost:%d", port)
}
if project.Environment == "cloud" && len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
// Overrides all the things
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
// Noproxy as it's a local request
topClient := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
streamUrl := fmt.Sprintf("%s/api/v1/streams", backendUrl)
log.Printf("[DEBUG][%s] Sending User Input result to self because we are on cloud env/action is skipped. URL: %#v", result.ExecutionId, streamUrl)
req, err := http.NewRequest(
"POST",
streamUrl,
bytes.NewBuffer([]byte(fullMarshal)),
)
if err != nil {
log.Printf("[ERROR] Failed creating request for stream during SKIPPED user input (1): %s", err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Execution (%s) action failed to skip. Contact support if this persists.", oldExecution.ExecutionId), errors.New("Execution action failed to skip. Contact support if this persists.")
}
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Failed sending request for stream during SKIPPED user input (2): %s", err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Execution (%s) action failed to skip during send. Contact support if this persists.", oldExecution.ExecutionId), errors.New("Execution action failed to skip during send. Contact support if this persists.")
}
defer newresp.Body.Close()
}
return workflowExecution, ExecInfo{}, fmt.Sprintf("Execution (%s) action skipped", oldExecution.ExecutionId), errors.New("User Input: Execution action skipped!")
}
foundresult = result
newResults = append(newResults, result)
} else {
newResults = append(newResults, result)
}
}
if foundresult.Action.AppName != "" {
// Should resend the result to redeploy the job?
log.Printf("[INFO][%s] Rerunning node for user input with WAITING", oldExecution.ExecutionId)
b, err := json.Marshal(foundresult)
if err != nil {
log.Printf("[WARNING][%s] Failed to run node for user input with WAITING", oldExecution.ExecutionId)
} else {
ResendActionResult(b, 4)
}
} else {
log.Printf("[WARNING][%s] No job to rerun for user input as a WAITING node was not found", oldExecution.ExecutionId)
return workflowExecution, ExecInfo{}, "", errors.New("Already Clicked")
}
// Add new execution to queue?
//if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
return *oldExecution, ExecInfo{}, "", errors.New("User Input")
}
if referenceok {
log.Printf("[DEBUG] Handling an old execution continuation! Start: %s", start)
// Will use the old name, but still continue with NEW ID
oldExecution, err := GetWorkflowExecution(ctx, referenceId[0])
if err != nil {
log.Printf("[ERROR][%s] Failed getting execution (execution): %s", referenceId[0], err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Failed getting execution ID '%s' because it doesn't exist (prepare exec).", referenceId[0]), err
}
if oldExecution.Status != "WAITING" {
return workflowExecution, ExecInfo{}, "", errors.New("Workflow is no longer with status waiting. Can't continue.")
}
if startok {
for _, result := range oldExecution.Results {
if result.Action.ID == start[0] {
if result.Status == "SUCCESS" || result.Status == "FINISHED" {
// Disabling this to allow multiple continuations
//return WorkflowExecution{}, ExecInfo{}, "", errors.New("This workflow has already been continued")
}
//log.Printf("Start: %s", result.Status)
}
}
}
workflowExecution = *oldExecution
// A previously stopped workflow. Same priority as subflow.
workflowExecution.Priority = 9
}
if len(workflowExecution.ExecutionId) == 0 {
sessionToken := uuid.NewV4()
workflowExecution.ExecutionId = sessionToken.String()
} else {
log.Printf("[DEBUG] Using the same executionId as before: %s", workflowExecution.ExecutionId)
makeNew = false
}
// Don't override workflow defaults
}
//log.Printf("[DEBUG][%s] STARTING IF/ELSE NODE REMAPPING", workflowExecution.ExecutionId)
for branchIndex, branch := range workflowExecution.Workflow.Branches {
if len(branch.SourceParent) == 0 {
continue
}
elseCondition := false
if strings.HasSuffix(branch.SourceParent, "-else") {
branch.SourceParent = strings.TrimSuffix(branch.SourceParent, "-else")
elseCondition = true
}
parentAction := Action{}
for _, workflowAction := range workflowExecution.Workflow.Actions {
if workflowAction.ID == branch.SourceParent {
parentAction = workflowAction
break
}
}
if parentAction.ID == "" {
continue
}
actionLabelParsed := fmt.Sprintf("$%s.%s.valid", strings.ToLower(strings.ReplaceAll(parentAction.Label, " ", "_")), branch.SourceID)
// FIXME: Add condition to them:
// This is going to check the result for:
// $parentname.${branch.SourceID}.valid == True
// For the else, add one for ALL others:
// $parentname.${branch.SourceID}.valid == False & $parentname.${branch.SourceID2}.valid == False && $parentname.${branch.SourceID3}.valid == False
//log.Printf("[DEBUG] Branch REMAP: %s -> %s: %#v", branch.SourceID, branch.DestinationID, branch)
workflowExecution.Workflow.Branches[branchIndex].SourceID = branch.SourceParent
newCondition := Condition{
Source: WorkflowAppActionParameter{
Value: actionLabelParsed,
ActionField: "",
ID: uuid.NewV4().String(),
Name: "source",
Variant: "STATIC_VALUE",
},
Condition: WorkflowAppActionParameter{
Value: "equals",
ID: uuid.NewV4().String(),
Name: "condition",
Variant: "STATIC_VALUE",
},
Destination: WorkflowAppActionParameter{
Value: "true",
ActionField: "",
ID: uuid.NewV4().String(),
Name: "destination",
Variant: "STATIC_VALUE",
},
}
if elseCondition {
newCondition.Source.Value = fmt.Sprintf("$%s.run_else", strings.ToLower(strings.ReplaceAll(parentAction.Label, " ", "_")))
}
workflowExecution.Workflow.Branches[branchIndex].Conditions = append(workflowExecution.Workflow.Branches[branchIndex].Conditions, newCondition)
// Changed to use success: false -> else
//newCondition.Destination.Value = "false"
//elseCondition = append(elseCondition, newCondition)
}
//log.Printf("[DEBUG][%s] ENDING IF/ELSE NODE REMAPPING", workflowExecution.ExecutionId)
if workflowExecution.SubExecutionCount == 0 {
workflowExecution.SubExecutionCount = 1
}
if workflowExecution.SubExecutionCount >= maxExecutionDepth {
return workflowExecution, ExecInfo{}, fmt.Sprintf("Max subflow of %d reached", maxExecutionDepth), err
}
if workflowExecution.Priority == 0 {
workflowExecution.Priority = 10
}
if startok {
//workflowExecution.Workflow.Start = start[0]
workflowExecution.Start = start[0]
} else {
if workflowExecution.ExecutionSource == "schedule" {
// This is kind of silly as it doesn't really check which trigger
// it instead just picks one (meaning we can max have one..?)
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.TriggerType != "SCHEDULE" {
continue
}
startChanged := false
for _, branch := range workflowExecution.Workflow.Branches {
if branch.SourceID != trigger.ID {
continue
}
if workflowExecution.Start != branch.DestinationID {
workflowExecution.Start = branch.DestinationID
startChanged = true
break
}
}
if startChanged {
break
}
}
}
}
// FIXME - regex uuid, and check if already exists?
if len(workflowExecution.ExecutionId) != 36 {
log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId)
return workflowExecution, ExecInfo{}, "Invalid uuid", err
}
// FIXME - find owner of workflow
// FIXME - get the actual workflow itself and build the request
// MAYBE: Don't send the workflow within the pubsub, as this requires more data to be sent
// Check if a worker already exists for company, else run one with:
// locations, project IDs and subscription names
// When app is executed:
// Should update with status execution (somewhere), which will trigger the next node
// IF action.type == internal, we need the internal watcher to be running and executing
// This essentially means the WORKER has to be the responsible party for new actions in the INTERNAL landscape
// Results are ALWAYS posted back to cloud@execution_id?
if makeNew {
workflowExecution.Type = "workflow"
//workflowExecution.Stream = "tmp"
//workflowExecution.WorkflowQueue = "tmp"
//workflowExecution.SubscriptionNameNodestream = "testcompany-nodestream"
//workflowExecution.Locations = []string{"europe-west2"}
//workflowExecution.ProjectId = gceProject
workflowExecution.WorkflowId = workflow.ID
workflowExecution.StartedAt = int64(time.Now().Unix())
workflowExecution.CompletedAt = 0
workflowExecution.Authorization = uuid.NewV4().String()
// Status for the entire workflow.
workflowExecution.Status = "EXECUTING"
}
if len(workflowExecution.ExecutionSource) == 0 {
//log.Printf("[INFO] No execution source (trigger) specified. Setting to default")
workflowExecution.ExecutionSource = "default"
}
// Look for header 'appauth' with upper/lowercase check
authHeader := ""
chosenEnvironment := ""
for key, value := range request.Header {
if strings.ToLower(key) == "appauth" {
authHeader = value[0]
}
if strings.ToLower(key) == "environment" {
chosenEnvironment = value[0]
}
}
// curl 'http://localhost:5002/api/v1/workflows/{workflow_id}/run' -H 'app_auth: appname=auth_id;appname2=auth_id2'
authGroups := []AppAuthenticationGroup{}
allAuths := []AppAuthenticationStorage{}
if len(authHeader) > 0 {
//log.Printf("\n\n\n\n[DEBUG] Found appauth header in request. Attempting to find matching auth with name/ID '%s'\n\n\n\n", authHeader)
if len(allAuths) == 0 {
allAuths, err = GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting all app authentications: %s", err)
}
}
appAuthSplit := strings.Split(authHeader, ";")
for _, authitem := range appAuthSplit {
authitemSplit := strings.Split(authitem, "=")
if len(authitemSplit) != 2 {
continue
}
// Find the app in the workflow and replace the ID
appname := strings.ReplaceAll(strings.ToLower(strings.TrimSpace(authitemSplit[0])), " ", "_")
authId := strings.ReplaceAll(strings.TrimSpace(authitemSplit[1]), " ", "_")
authFound := false
for _, auth := range allAuths {
if auth.Id == authId || strings.ReplaceAll(auth.Label, " ", "_") == authId {
authFound = true
authId = auth.Id
}
}
if !authFound {
return workflowExecution, ExecInfo{}, fmt.Sprintf("App auth not found: %s", authId), errors.New(fmt.Sprintf("App auth '%s' not found for app '%s'", authId, appname))
}
found := false
for actionIndex, action := range workflowExecution.Workflow.Actions {
if strings.ReplaceAll(strings.ToLower(action.AppName), " ", "_") == appname || strings.ReplaceAll(strings.ToLower(action.ID), " ", "_") == appname || strings.ReplaceAll(strings.ToLower(action.AppID), " ", "_") == appname {
workflowExecution.Workflow.Actions[actionIndex].AuthenticationId = authId
found = true
}
}
if !found {
log.Printf("[DEBUG][%s] Didn't find custom app/auth: %s", workflowExecution.ExecutionId, appname)
}
}
}
workflowExecution.ExecutionVariables = workflow.ExecutionVariables
if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 {
workflowExecution.Start = workflowExecution.Workflow.Start
}
startnodeFound := false
newStartnode := ""
for actionIndex, item := range workflowExecution.Workflow.Actions {
if item.ID == workflowExecution.Start {
startnodeFound = true
}
// Backup fallback in case we can't find the one assigned
if item.IsStartNode {
newStartnode = item.ID
}
// Fix names of parameters
for paramIndex, param := range item.Parameters {
if param.Name == "headers" {
// Check if it's a JSON object
param.Value = strings.TrimSpace(param.Value)
if strings.HasPrefix(param.Value, "{") && strings.HasSuffix(param.Value, "}") {
// Try to map it with key:value
newheaders := ""
var headers map[string]string
err := json.Unmarshal([]byte(param.Value), &headers)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling headers: %s", err)
} else {
for key, value := range headers {
newheaders += fmt.Sprintf("%s: %s\n", key, value)
}
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = newheaders
continue
}
}
// Special weird mistake cases in apps without newlines
param.Value = strings.ReplaceAll(param.Value, "application/jsonContent-Type", "application/json\nContent-Type")
param.Value = strings.ReplaceAll(param.Value, "application/jsonAccept", "application/json\nAccept")
}
if param.Name == "queries" {
// Check if it's a JSON object
param.Value = strings.TrimSpace(param.Value)
if strings.HasPrefix(param.Value, "{") && strings.HasSuffix(param.Value, "}") {
// Try to map it with key:value
newqueries := ""
var queries map[string]string
err := json.Unmarshal([]byte(param.Value), &queries)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling queries: %s", err)
} else {
for key, value := range queries {
newqueries += fmt.Sprintf("%s=%s&", key, value)
}
// Remove trailing & if exists
if len(newqueries) > 0 {
newqueries = newqueries[:len(newqueries)-1]
}
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = newqueries
continue
}
}
}
// Added after problem with api-secret -> apisecret
if strings.Contains(param.Description, "header") {
if strings.Contains(param.Value, "=undefined") {
newheaders := []string{}
for _, line := range strings.Split(param.Value, "\n") {
if !strings.Contains(line, "=undefined") {
newheaders = append(newheaders, line)
continue
}
}
item.Parameters[paramIndex].Value = strings.Join(newheaders, "\n")
}
continue
}
if strings.Contains(param.Description, "query") {
continue
}
newName := GetValidParameters([]string{param.Name})
if len(newName) > 0 {
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Name = newName[0]
}
}
}
if !startnodeFound {
if len(newStartnode) > 0 {
workflowExecution.Start = newStartnode
} else {
log.Printf("[WARNING][%s] Couldn't find startnode %s among %d actions in workflow '%s'. ATEEMPTED remap to '%s'", workflowExecution.ExecutionId, workflowExecution.Start, len(workflowExecution.Workflow.Actions), workflowExecution.Workflow.ID, newStartnode)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Startnode couldn't be found"), errors.New("Startnode isn't defined in this workflow..")
}
}
workflowExecution.Workflow.Validation = TypeValidation{}
childNodes := FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{})
//topic := "workflows"
startFound := false
// FIXME - remove this?
newActions := []Action{}
defaultResults := []ActionResult{}
if project.Environment == "cloud" {
//apps, err := GetPrioritizedApps(ctx, user)
//if err != nil {
// log.Printf("[WARNING] Error: Failed getting apps during setup: %s", err)
//}
}
// Overwrites environment before validation below
if len(chosenEnvironment) > 0 {
for actionIndex, _ := range workflowExecution.Workflow.Actions {
workflowExecution.Workflow.Actions[actionIndex].Environment = chosenEnvironment
}
}
isAuthgroup := false
if request != nil {
// Check if "authgroups" param exists
if authGroups, authGroupsOk := request.URL.Query()["authgroups"]; authGroupsOk {
if len(authGroups) > 0 && authGroups[0] == "true" {
isAuthgroup = true
workflowExecution.ExecutionSource = "authgroups"
}
}
}
org := &Org{}
previousEnvironment := ""
orgEnvironments := []Environment{}
subExecutionsDone := false
for workflowExecutionIndex, action := range workflowExecution.Workflow.Actions {
//action.LargeImage = ""
if action.ID == workflowExecution.Start {
startFound = true
}
// Fill in apikey?
if project.Environment == "cloud" {
if (action.AppName == "Shuffle Tools" || action.AppName == "email") && action.Name == "send_email_shuffle" || action.Name == "send_sms_shuffle" {
for paramKey, param := range action.Parameters {
if param.Name == "apikey" && action.Name == "send_sms_shuffle" {
// This will be in cache after running once or twice AKA fast
org, err := GetOrg(ctx, workflow.OrgId)
if err != nil {
log.Printf("[ERROR] Error getting org in APIkey replacement: %s", err)
continue
}
// Make sure to find one that's belonging to the org
// Picking random last user if
backupApikey := ""
for _, user := range org.Users {
if len(user.ApiKey) == 0 {
continue
}
if user.Role != "org-reader" {
backupApikey = user.ApiKey
}
if len(user.Orgs) == 1 || user.ActiveOrg.Id == workflowExecution.Workflow.OrgId {
//log.Printf("Choice: %s, %s - %s", user.Username, user.Id, user.ApiKey)
workflowExecution.Workflow.Actions[workflowExecutionIndex].Parameters[paramKey].Value = user.ApiKey
break
}
}
if len(action.Parameters[paramKey].Value) == 0 {
log.Printf("[WARNING] No apikey user found. Picking first random user")
action.Parameters[paramKey].Value = backupApikey
workflowExecution.Workflow.Actions[workflowExecutionIndex].Parameters[paramKey].Value = backupApikey
}
if debug {
log.Printf("[DEBUG] Replaced apikey for %s with %s", param.Name, action.Parameters[paramKey].Value)
}
break
}
// Autoreplace in general, even if there is a key. Overwrite previous configs to ensure this becomes the norm. Frontend also matches.
if param.Name != "apikey" {
//log.Printf("Autoreplacing apikey")
// This will be in cache after running once or twice AKA fast
org, err = GetOrg(ctx, workflowExecution.Workflow.OrgId)
if err != nil {
log.Printf("[ERROR] Error getting org in APIkey replacement: %s", err)
continue
}
// Make sure to find one that's belonging to the org
// Picking random last user if
backupApikey := ""
for _, user := range org.Users {
if len(user.ApiKey) == 0 {
continue
}
if user.Role != "org-reader" {
backupApikey = user.ApiKey
}
if (len(user.Orgs) == 1 || user.ActiveOrg.Id == workflowExecution.Workflow.OrgId) && action.Name != "send_email_shuffle" {
//log.Printf("Choice: %s, %s - %s", user.Username, user.Id, user.ApiKey)
action.Parameters[paramKey].Value = user.ApiKey
break
}
}
if len(action.Parameters[paramKey].Value) == 0 {
log.Printf("[WARNING] No apikey user found. Picking first random user")
action.Parameters[paramKey].Value = backupApikey
}
break
}
}
}
}
if action.Environment == "" {
// Fallback automatically to the project environment
if len(previousEnvironment) > 0 {
action.Environment = previousEnvironment
} else if project.Environment == "cloud" {
action.Environment = "cloud"
}
if len(action.Environment) == 0 {
log.Printf("[ERROR] Environment is not defined for action %s in workflow %s (%s)", action.Name, workflowExecution.Workflow.Name, workflowExecution.Workflow.ID)
action.Environment = "shuffle"
}
if len(action.Environment) > 0 {
previousEnvironment = action.Environment
}
} else {
previousEnvironment = action.Environment
}
if action.AuthenticationId == "authgroups" && !subExecutionsDone && workflowExecution.ExecutionSource != "authgroup" && !isAuthgroup {
// FIXME: Check if this action IS under the startnode or not
childNodes := FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{})
if workflowExecution.Start != action.ID && !ArrayContains(childNodes, action.ID) {
log.Printf("[DEBUG][%s] Skipping action %s as it's not under the startnode, and uses authgroups", workflowExecution.ExecutionId, action.Label)
continue
}
discoveredApikey := ""
if len(org.Users) == 0 {
org, err = GetOrg(ctx, workflowExecution.Workflow.OrgId)
if err != nil {
log.Printf("[ERROR] Failed getting org for action %s: %s", action.Label, err)
}
}
if len(org.Users) != 0 {
log.Printf("[DEBUG][%s] Found %d users in org %s", workflowExecution.ExecutionId, len(org.Users), org.Id)
for _, user := range org.Users {
if user.Role != "admin" {
continue
}
foundUser, err := GetUser(ctx, user.Id)
if err != nil {
log.Printf("[ERROR] Failed getting user %s: %s", user.Id, err)
}
if len(foundUser.ApiKey) == 0 {
continue
}
log.Printf("[DEBUG][%s] Found apikey for user %s to be used for subexecutions", workflowExecution.ExecutionId, foundUser.Username)
discoveredApikey = foundUser.ApiKey
break
}
}
log.Printf("[DEBUG][%s] Found authgroups in action %s (%s)", workflowExecution.ExecutionId, action.Label, action.ID)
if len(authGroups) == 0 {
authGroups, err = GetAuthGroups(ctx, workflow.OrgId)
if err != nil {
log.Printf("[ERROR] Failed getting authgroups for org %s: %s", workflow.OrgId, err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Failed getting authgroups for org %s: %s", workflow.OrgId, err), err
}
if len(authGroups) == 0 {
log.Printf("[ERROR] No authgroups found for org %s", workflow.OrgId)
return workflowExecution, ExecInfo{}, fmt.Sprintf("No authgroups found for org %s", workflow.OrgId), errors.New("No authgroups exist. Create them by going to: /admin?tab=app_auth")
}
}
relevantAuthgroups := []AppAuthenticationGroup{}
for _, authGroup := range workflow.AuthGroups {
found := false
for _, group := range authGroups {
if group.Id == authGroup {
relevantAuthgroups = append(relevantAuthgroups, group)
found = true
break
}
}
if !found {
log.Printf("[WARNING] Authgroup %s not found in org %s", authGroup, workflow.OrgId)
}
}
if len(relevantAuthgroups) == 0 {
log.Printf("[ERROR] No relevant authgroups found for org %s. Do they still exist?", workflow.OrgId)
return workflowExecution, ExecInfo{}, fmt.Sprintf("No relevant authgroups found for org %s. Do they still exist?", workflow.OrgId), errors.New("No relevant authgroups found for workflow. Do they still exist?")
}
log.Printf("[DEBUG][%s] Found %d relevant auth groups for action %s", workflowExecution.ExecutionId, len(relevantAuthgroups), action.Label)
// FIXME: Start doing replication here.
// First request (this one) should use the first authgroup
// Second and all after should run as new requests that are overwritten with the right auth.
firstGroup := relevantAuthgroups[0]
for authgroupindex, authgroup := range relevantAuthgroups {
if len(orgEnvironments) == 0 {
orgEnvironments, err = GetEnvironments(ctx, workflowExecution.Workflow.OrgId)
if err != nil {
log.Printf("[ERROR] Failed getting environments for org %s: %s", workflow.OrgId, err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Failed getting environments for org %s: %s", workflow.OrgId, err), err
}
}
environmentFound := false
if authgroup.Environment == "" {
// Fallback automatically to the project environment
for _, env := range orgEnvironments {
if env.Default {
authgroup.Environment = env.Name
}
}
}
for _, env := range orgEnvironments {
if strings.ToLower(env.Name) == strings.ToLower(authgroup.Environment) || env.Id == authgroup.Environment {
environmentFound = true
break
}
}
if !environmentFound {
log.Printf("[ERROR] Environment %s not found for authgroup %s", authgroup.Environment, authgroup.Id)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Environment %s not found for authgroup %s", authgroup.Environment, authgroup.Id), errors.New(fmt.Sprintf("Environment %s not found for authgroup %s", authgroup.Environment, authgroup.Id))
}
for findActionIndex, findAction := range workflowExecution.Workflow.Actions {
workflowExecution.Workflow.Actions[findActionIndex].Environment = authgroup.Environment
if findAction.AuthenticationId != "authgroups" {
continue
}
log.Printf("[DEBUG][%s] Found authgroups in action %s (%s)", workflowExecution.ExecutionId, findAction.Label, findAction.ID)
// Find the app in the group
authFound := false
for _, auth := range firstGroup.AppAuths {
if strings.ToLower(auth.App.Name) == strings.ToLower(findAction.AppName) || auth.App.ID == findAction.AppID {
workflowExecution.Workflow.Actions[findActionIndex].AuthenticationId = auth.Id
if workflowExecution.Workflow.Actions[findActionIndex].ID == action.ID {
action = workflowExecution.Workflow.Actions[findActionIndex]
}
authFound = true
break
}
}
log.Printf("[DEBUG][%s] New auth ID for %s: %s", workflowExecution.ExecutionId, findAction.AppName, workflowExecution.Workflow.Actions[findActionIndex].AuthenticationId)
if !authFound {
log.Printf("[ERROR][%s] App %s not found in authgroup %s", workflowExecution.ExecutionId, findAction.AppName, firstGroup.Id)
return workflowExecution, ExecInfo{}, fmt.Sprintf("App %s not found in authgroup %s", findAction.AppName, firstGroup.Id), errors.New(fmt.Sprintf("App %s not found in authgroup %s", findAction.AppName, firstGroup.Id))
}
//action = workflowExecution.Workflow.Actions[workflowExecutionIndex]
log.Printf("[DEBUG][%s] Updated action with ID %s to use authgroup %s", workflowExecution.ExecutionId, action.ID, firstGroup.Id)
}
if authgroupindex == 0 {
// First one is the current execution.
// No need to run as subflow
workflowExecution.Authgroup = authgroup.Label
continue
}
// Replication starts here
log.Printf("[DEBUG][%s] SHOULD REPLICATE AUTHGROUP ACTION MAPPING into %d groups", workflowExecution.ExecutionId, len(relevantAuthgroups))
subExecutionsDone = true
// Send a self-request to execute THIS workflow as a subflow
go executeAuthgroupSubflow(workflowExecution, authgroup, discoveredApikey)
/*
err = executeAuthgroupSubflow(workflowExecution, authgroup, discoveredApikey)
if err != nil {
log.Printf("[ERROR] Failed executing authgroup subflow: %s", err)
}
*/
}
//log.Printf("[DEBUG][%s] RETURNING BEFORE ORIGINAL SUBFLOW CAN RUN", workflowExecution.ExecutionId)
}
if len(action.AuthenticationId) > 0 {
if len(allAuths) == 0 {
allAuths, err = GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id)
if err != nil {
log.Printf("[ERROR] Api authentication failed in get all app auth for ID %s: %s", workflow.ExecutingOrg.Id, err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Api authentication failed in get all app auth for %s: %s", workflow.ExecutingOrg.Id, err), err
}
}
// Simplified it all into a single function
action, workflowExecution = GetAuthentication(ctx, workflowExecution, action, allAuths)
//action.Parameters = newParams
}
action.LargeImage = ""
if len(action.Label) == 0 {
action.Label = action.ID
}
//log.Printf("LABEL: %s", action.Label)
newActions = append(newActions, action)
// If the node is NOT found, it's supposed to be set to SKIPPED,
// as it's not a childnode of the startnode
// This is a configuration item for the workflow itself.
if len(workflowExecution.Results) > 0 {
extra := 0
for _, trigger := range workflowExecution.Workflow.Triggers {
//log.Printf("Appname trigger (0): %s", trigger.AppName)
if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
extra += 1
}
}
} else if len(workflowExecution.Results) == 0 && !workflowExecution.Workflow.Configuration.StartFromTop {
found := false
for _, nodeId := range childNodes {
if nodeId == action.ID {
//log.Printf("Found %s", action.ID)
found = true
}
}
if !found {
if action.ID == workflowExecution.Start {
continue
}
curaction := Action{
AppName: action.AppName,
AppVersion: action.AppVersion,
Label: action.Label,
Name: action.Name,
ID: action.ID,
}
defaultResults = append(defaultResults, ActionResult{
Action: curaction,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: `{"success": false, "reason": "Skipped because it's not under the startnode (1)"}`,
StartedAt: 0,
CompletedAt: 0,
Status: "SKIPPED",
})
}
}
}
// Added fixes for e.g. URL's ending in /
fixes := []string{"url"}
for actionIndex, action := range workflowExecution.Workflow.Actions {
if strings.ToLower(action.AppName) == "http" {
continue
}
for paramIndex, param := range action.Parameters {
if !param.Configuration {
continue
}
if ArrayContains(fixes, strings.ToLower(param.Name)) {
if strings.HasSuffix(param.Value, "/") {
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = param.Value[0 : len(param.Value)-1]
}
}
}
}
// Not necessary with comments at all
workflowExecution.Workflow.Comments = []Comment{}
for _, trigger := range workflowExecution.Workflow.Triggers {
//log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start)
if trigger.ID == workflowExecution.Start {
if trigger.AppName == "User Input" {
startFound = true
break
}
}
if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
found := false
for _, node := range childNodes {
if node == trigger.ID {
found = true
break
}
}
if !found {
curaction := Action{
AppName: "shuffle-subflow",
AppVersion: trigger.AppVersion,
Label: trigger.Label,
Name: trigger.Name,
ID: trigger.ID,
}
found := false
for _, res := range defaultResults {
if res.Action.ID == trigger.ID {
found = true
break
}
}
if !found {
defaultResults = append(defaultResults, ActionResult{
Action: curaction,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: `{"success": false, "reason": "Skipped because it's not under the startnode (2)"}`,
StartedAt: 0,
CompletedAt: 0,
Status: "SKIPPED",
})
}
} else {
// Replaces trigger with the subflow
}
}
}
if !startFound {
if len(workflowExecution.Start) == 0 && len(workflowExecution.Workflow.Start) > 0 {
workflowExecution.Start = workflow.Start
} else if len(workflowExecution.Workflow.Actions) > 0 {
workflowExecution.Start = workflowExecution.Workflow.Actions[0].ID
} else {
log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start))
}
}
// Validation of SKIPPED nodes
if len(workflowExecution.Start) > 0 {
childNodes := FindChildNodes(workflowExecution.Workflow, workflowExecution.Start, []string{}, []string{})
//log.Printf("\n\n\n[DEBUG][%s] STARTUP NODES UNDER '%s' (%d): %#v. Total actions: %#v\n\n\n", workflowExecution.ExecutionId, workflowExecution.Start, len(childNodes), childNodes, len(workflowExecution.Workflow.Actions))
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == workflowExecution.Start {
continue
}
if ArrayContains(childNodes, action.ID) {
continue
}
foundResult := false
for _, result := range defaultResults {
if result.Action.ID == action.ID {
foundResult = true
break
}
}
if !foundResult {
defaultResults = append(defaultResults, ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: `{"success": false, "reason": "Skipped because it's not under the startnode (3)"}`,
StartedAt: 0,
CompletedAt: 0,
Status: "SKIPPED",
})
}
}
}
// Verification for execution environments
workflowExecution.Results = defaultResults
workflowExecution.Workflow.Actions = newActions
onpremExecution := true
environments := []string{}
if len(workflowExecution.ExecutionOrg) == 0 && len(workflow.ExecutingOrg.Id) > 0 {
workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id
}
var allEnvs []Environment
if len(workflowExecution.ExecutionOrg) > 0 {
//log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg)
allEnvironments, err := GetEnvironments(ctx, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("[ERROR][%s] Failed finding environments for %s: %s", workflowExecution.ExecutionId, workflowExecution.ExecutionOrg, err)
return workflowExecution, ExecInfo{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org"))
}
for _, curenv := range allEnvironments {
if curenv.Archived {
continue
}
allEnvs = append(allEnvs, curenv)
}
} else {
log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID)
return workflowExecution, ExecInfo{}, "No org identified for execution", errors.New("No org identified for execution")
}
if len(allEnvs) == 0 {
log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg)
return workflowExecution, ExecInfo{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg))
}
// Check if the actions are children of the startnode?
imageNames := []string{}
cloudExec := false
prevEnvironment := ""
for actionIndex, action := range workflowExecution.Workflow.Actions {
// Verify if the action environment exists and append
found := false
for _, env := range allEnvs {
if strings.ToLower(env.Name) != strings.ToLower(action.Environment) {
continue
}
found = true
if env.Type == "cloud" || strings.ToLower(env.Name) == "cloud" {
cloudExec = true
} else if env.Type == "onprem" {
onpremExecution = true
} else {
log.Printf("[ERROR] No handler for environment type %s", env.Type)
return workflowExecution, ExecInfo{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type))
}
break
}
if !found {
if action.Environment == "Shuffle" && project.Environment == "cloud" {
action.Environment = "Cloud"
workflowExecution.Workflow.Actions[actionIndex].Environment = "Cloud"
cloudExec = true
} else {
if project.Environment == "cloud" {
action.Environment = "Cloud"
workflowExecution.Workflow.Actions[actionIndex].Environment = "Cloud"
cloudExec = true
} else {
action.Environment = "Shuffle"
workflowExecution.Workflow.Actions[actionIndex].Environment = "Shuffle"
}
}
if len(prevEnvironment) > 0 {
action.Environment = prevEnvironment
workflowExecution.Workflow.Actions[actionIndex].Environment = prevEnvironment
}
} else {
prevEnvironment = action.Environment
}
found = false
for _, env := range environments {
if env == action.Environment {
found = true
break
}
}
// Check if the app exists?
newName := action.AppName
newName = strings.Replace(newName, " ", "-", -1)
imageNames = append(imageNames, fmt.Sprintf("%s:%s_%s", baseDockerName, newName, action.AppVersion))
if !found {
environments = append(environments, action.Environment)
}
}
if len(workflowExecution.Workflow.ExecutingOrg.Id) == 0 || workflowExecution.ExecutionOrg != workflowExecution.Workflow.ExecutingOrg.Id {
workflowExecution.Workflow.ExecutingOrg = OrgMini{
Id: workflowExecution.ExecutionOrg,
}
}
workflowExecution.Workflow.OrgId = workflowExecution.Workflow.ExecutingOrg.Id
// Means executing a subflow is happening
if len(workflowExecution.ExecutionParent) > 0 {
go IncrementCache(ctx, workflowExecution.ExecutionOrg, "subflow_executions")
}
// NEW check for subflow
// This is also handling triggers -> action translation now for subflow
extra := 0
newTriggers := []Trigger{}
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.TriggerType != "SUBFLOW" && trigger.TriggerType != "USERINPUT" {
newTriggers = append(newTriggers, trigger)
continue
}
if trigger.TriggerType == "SUBFLOW" {
//log.Printf("[INFO] Subflow trigger found during execution! envs: %#v", environments)
// Find branch that has the subflow as destinationID
foundenv := ""
for _, branch := range workflowExecution.Workflow.Branches {
if branch.DestinationID == trigger.ID {
// FIX: May not work for subflow -> subflow if they are added in opposite order or something weird
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == branch.SourceID {
foundenv = action.Environment
break
}
}
if len(foundenv) > 0 {
break
}
}
}
// Backup env :>
if len(foundenv) == 0 && len(environments) > 0 {
//log.Printf("[ERROR] Fallback to environment %s for subflow (default). Does it still run?", environments[0])
foundenv = environments[0]
}
// Setting to default?
// environments := []string{}
action := GetAction(workflowExecution, trigger.ID, foundenv)
action.Label = trigger.Label
action.ID = trigger.ID
action.Name = "run_subflow"
action.AppName = "shuffle-subflow"
action.AppVersion = "1.1.0"
action.Parameters = []WorkflowAppActionParameter{}
for _, parameter := range trigger.Parameters {
parameter.Variant = "STATIC_VALUE"
if parameter.Name == "user_apikey" {
continue
}
action.Parameters = append(action.Parameters, parameter)
//log.Printf("[INFO] Adding parameter %s to subflow", parameter.Name)
}
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_workflow",
Value: workflowExecution.Workflow.ID,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_execution",
Value: workflowExecution.ExecutionId,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_auth",
Value: workflowExecution.Authorization,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "user_apikey",
Value: workflowExecution.Authorization,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_node",
Value: action.ID,
})
backendUrl := os.Getenv("BASE_URL")
/*
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
*/
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 && strings.Contains(os.Getenv("SHUFFLE_CLOUDRUN_URL"), "http") {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
if len(backendUrl) > 0 {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "backend_url",
Value: backendUrl,
})
} else {
log.Printf("[ERROR] No Backend URL found for subflow. May fail to connect properly.")
}
workflowExecution.Workflow.Actions = append(workflowExecution.Workflow.Actions, action)
} else {
newTriggers = append(newTriggers, trigger)
extra += 1
}
}
workflowExecution.Workflow.Triggers = newTriggers
// Checking authentication fields as they should now be filled in no matter where
if len(workflowExecution.ExecutionOrg) == 0 {
log.Printf("\n\n[ERROR] No org found for execution. This should not happen.\n\n")
}
if len(org.Id) == 0 && workflowExecution.ExecutionOrg != "INTERNAL" {
org, err = GetOrg(ctx, workflowExecution.ExecutionOrg)
if err != nil {
log.Printf("[ERROR] Failed to get org %#v (workflow exec): %s", workflowExecution.ExecutionOrg, err)
}
}
// Clear out example & description fields
for actionIndex, action := range workflowExecution.Workflow.Actions {
for paramIndex, _ := range action.Parameters {
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Example = ""
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Description = ""
}
}
// A way to set default config for kmsid if it's not set
if len(org.Defaults.KmsId) == 0 {
if len(allAuths) == 0 {
allAuths, err = GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed to get auths during kms prep: %s", err)
}
}
for _, auth := range allAuths {
if strings.ReplaceAll(strings.TrimSpace(strings.ToLower(auth.Label)), "_", " ") == "kms shuffle storage" {
org.Defaults.KmsId = auth.Id
break
}
}
}
if len(org.Defaults.KmsId) > 0 {
if len(allAuths) == 0 {
allAuths, err = GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed to get auths during kms prep: %s", err)
}
}
foundAuth := AppAuthenticationStorage{}
for _, auth := range allAuths {
if auth.Id != org.Defaults.KmsId {
continue
}
foundAuth = auth
break
}
// Use the auth to decrypt
if foundAuth.Id == org.Defaults.KmsId {
foundAuth.App.LargeImage = ""
foundAuth.App.SmallImage = ""
findKeys := []string{}
for actionIndex, action := range workflowExecution.Workflow.Actions {
for paramIndex, param := range action.Parameters {
// FIXME: Should we allow KMS for ANYthing?
if !param.Configuration && !kmsDebug {
continue
}
if strings.HasPrefix(param.Value, "/") {
param.Value = strings.TrimPrefix(param.Value, "/")
}
// Allow for both kms/ kms. and kms: as prefix
if !strings.HasPrefix(strings.ToLower(param.Value), "kms.") && !strings.HasPrefix(strings.ToLower(param.Value), "kms/") && !strings.HasPrefix(strings.ToLower(param.Value), "kms:") {
continue
}
splitValue := "/"
if strings.Contains(strings.ToLower(param.Value), "kms:") {
splitValue = ":"
}
if strings.HasSuffix(param.Value, splitValue) {
param.Value = param.Value[0 : len(param.Value)-1]
}
if param.Configuration {
param.Value = fmt.Sprintf("%s%s${%s}", param.Value, splitValue, param.Name)
}
if !ArrayContains(findKeys, param.Value) {
findKeys = append(findKeys, param.Value)
}
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = param.Value
}
}
// Should run all keys goroutines, then go find them again when all are done and replace
// Wtf is this garbage
if len(findKeys) > 0 {
//log.Printf("\n\n\n\n[INFO] Found %d auth key(s) to decrypt from KMS\n\n\n\n", len(findKeys))
// Have to set the workflow exec in cache while running this so that access rights exist
foundValues := map[string]string{}
marshalledExec, err := json.Marshal(workflowExecution)
if err == nil {
cacheKey := fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId)
err = SetCache(ctx, cacheKey, marshalledExec, 1)
if err == nil {
// FIXME: Optimize this to run in parallel
// across multiple goroutines
for _, k := range findKeys {
decrypted, err := DecryptKMS(ctx, foundAuth, k, workflowExecution.Authorization, workflowExecution.ExecutionId)
if err == nil {
foundValues[k] = decrypted
} else {
CreateOrgNotification(
ctx,
fmt.Sprintf("Failed to decrypt KMS key '%s'", k),
fmt.Sprintf("Failed to decrypt KMS key '%s'. Error: %s", k, err),
fmt.Sprintf("/workflows/%s?execution_id=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId),
workflowExecution.ExecutionOrg,
true,
"MEDIUM",
"KMS_DECRYPT_FAILURE",
)
}
}
} else {
log.Printf("[ERROR] Failed to set workflow execution in cache: %s", err)
}
} else {
log.Printf("[ERROR] Failed to marshal workflow execution for cache: %s", err)
}
// Continue here
if len(foundValues) > 0 {
for actionIndex, action := range workflowExecution.Workflow.Actions {
for paramIndex, param := range action.Parameters {
if !param.Configuration && !kmsDebug {
continue
}
if !strings.HasPrefix(strings.ToLower(param.Value), "kms.") && !strings.HasPrefix(strings.ToLower(param.Value), "kms/") && !strings.HasPrefix(strings.ToLower(param.Value), "kms:") {
continue
}
if val, ok := foundValues[param.Value]; ok {
//log.Printf("[INFO] Replacing value for %s with %s", param.Value, val)
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = val
} else {
// Remove the last /${%s} part if it exists in a key
for mapKey, mapValue := range foundValues {
if strings.HasPrefix(mapKey, param.Value) {
workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = mapValue
break
}
}
}
}
}
}
}
} else {
//log.Printf("[ERROR] Default KMS ID not found in organization. Will not be able to decrypt secrets.")
}
}
// Special Action cleanup in case authentication etc has gone wrong
// FIXME: Focused on URL field primarily
for actionIndex, _ := range workflowExecution.Workflow.Actions {
found := []string{}
newparams := []WorkflowAppActionParameter{}
for paramIndex, _ := range workflowExecution.Workflow.Actions[actionIndex].Parameters {
param := workflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex]
if !ArrayContains(found, param.Name) {
newparams = append(newparams, param)
found = append(found, param.Name)
continue
}
// Replaces the field
for existingParamIndex, _ := range newparams {
if param.Name != newparams[existingParamIndex].Name {
continue
}
// Special for urls
if param.Name == "url" && strings.Contains(param.Value, "http") && !strings.Contains(newparams[existingParamIndex].Value, "http") {
newparams[existingParamIndex] = param
}
break
}
log.Printf("[ERROR][%s] Duplicate Field in Action: %#v", workflowExecution.ExecutionId, param.Name)
}
workflowExecution.Workflow.Actions[actionIndex].Parameters = newparams
}
// Handles org setting for subflows
if len(workflowExecution.Workflow.ExecutingOrg.Name) == 0 {
// Maybe should be set from the parentorg?
if parentExecution.Workflow.ExecutingOrg.Id != "" {
workflowExecution.Workflow.ExecutingOrg = parentExecution.Workflow.ExecutingOrg
} else {
//log.Printf("[ERROR] Execution org name is empty, but should be filled in. This is a bug. Execution org: %+v", workflowExecution.ExecutionOrg)
workflowExecution.Workflow.ExecutingOrg.Name = org.Name
workflowExecution.Workflow.ExecutingOrg.Name = org.Id
}
}
if len(workflowExecution.Workflow.ID) > 0 {
workflowExecution.WorkflowId = workflowExecution.Workflow.ID
}
discoveredUser, authErr := HandleApiAuthentication(nil, request)
if authErr == nil && len(discoveredUser.Username) > 0 {
workflowExecution.Workflow.UpdatedBy = discoveredUser.Username
}
if workflowExecution.Workflow.Sharing == "form" || len(workflowExecution.Workflow.FormControl.InputMarkdown) > 0 {
//log.Printf("[DEBUG][%s] FORM RUN. Running Org injection AND liquid template removal", workflowExecution.ExecutionId)
// 1. Add Org-Id from the user to the existing workflowExecution.ExecutionArgument
validMap := map[string]interface{}{}
err := json.Unmarshal([]byte(workflowExecution.ExecutionArgument), &validMap)
if err != nil {
log.Printf("[ERROR][%s] Failed to unmarshal execution argument: %s. Instead mapping whole struct into exec", workflowExecution.ExecutionId, err)
validMap["exec"] = sanitizeString(workflowExecution.ExecutionArgument)
}
for key, value := range validMap {
if val, ok := value.(string); ok {
validMap[key] = sanitizeString(val)
}
}
// Overwriting it either way. Input NEEDS to be valid for map[string]interface{}{}
workflowExecution.ExecutionSource = "form"
if authErr != nil {
log.Printf("[ERROR] Failed to find user during form execution: %s", err)
} else {
validMap["form_type"] = "Manual form run. Less results returned."
validMap["org_id"] = discoveredUser.ActiveOrg.Id
marshalMap, err := json.Marshal(validMap)
if err != nil {
log.Printf("[ERROR] Failed to marshal execution argument: %s", err)
} else {
workflowExecution.ExecutionArgument = sanitizeString(string(marshalMap))
}
}
}
finished := ValidateFinished(ctx, extra, workflowExecution)
if finished {
log.Printf("[INFO][%s] Workflow already finished during startup. Is this correct?", workflowExecution.ExecutionId)
}
go DeleteCache(context.Background(), fmt.Sprintf("workflowexecution_%s", workflowExecution.WorkflowId))
go DeleteCache(context.Background(), fmt.Sprintf("workflowexecution_%s_50", workflowExecution.WorkflowId))
go DeleteCache(context.Background(), fmt.Sprintf("workflowexecution_%s_100", workflowExecution.WorkflowId))
// Force it into the database
return workflowExecution, ExecInfo{OnpremExecution: onpremExecution, Environments: environments, CloudExec: cloudExec, ImageNames: imageNames}, "", nil
}
func GetAuthentication(ctx context.Context, workflowExecution WorkflowExecution, action Action, allAuths []AppAuthenticationStorage) (Action, WorkflowExecution) {
if len(allAuths) == 0 {
return action, workflowExecution
}
workflow := workflowExecution.Workflow
curAuth := AppAuthenticationStorage{Id: ""}
authIndex := -1
for innerIndex, auth := range allAuths {
if auth.Id != action.AuthenticationId {
continue
}
authIndex = innerIndex
curAuth = auth
break
}
if len(curAuth.Id) == 0 {
log.Printf("[ERROR] App Auth ID %s doesn't exist for app '%s' among %d auth for org ID '%s'. Please re-authenticate the app (1).", action.AuthenticationId, action.AppName, len(allAuths), workflow.ExecutingOrg.Id)
workflowExecution.NotificationsCreated += 1
CreateOrgNotification(
ctx,
fmt.Sprintf("App Auth ID %s doesn't exist for app '%s' among %d auth for org ID '%s'", action.AuthenticationId, action.AppName, len(allAuths), workflow.ExecutingOrg.Id),
fmt.Sprintf("App Auth ID %s doesn't exist for app '%s' among %d auth for org ID '%s'. Please re-authenticate the app (2).", action.AuthenticationId, action.AppName, len(allAuths), workflow.ExecutingOrg.Id),
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, action.ID),
workflowExecution.ExecutionOrg,
true,
"HIGH",
"AUTH_ID_MISSING",
)
//return workflowExecution, ExecInfo{}, fmt.Sprintf("App Auth ID %s doesn't exist for app '%s' among %d auth for org ID '%s'. Please re-authenticate the app (1).", action.AuthenticationId, action.AppName, len(allAuths), workflow.ExecutingOrg.Id), errors.New(fmt.Sprintf("App Auth ID %s doesn't exist for app '%s' among %d auth for org ID '%s'. Please re-authenticate the app (2).", action.AuthenticationId, action.AppName, len(allAuths), workflow.ExecutingOrg.Id))
} else {
if curAuth.Encrypted {
setField := true
newFields := []AuthenticationStore{}
fieldLength := 0
for _, field := range curAuth.Fields {
parsedKey := fmt.Sprintf("%s_%d_%s_%s", curAuth.OrgId, curAuth.Created, curAuth.Label, field.Key)
newValue, err := HandleKeyDecryption([]byte(field.Value), parsedKey)
if err != nil {
if field.Key != "access_token" {
log.Printf("[ERROR][%s] Failed decryption (3) in auth org %s for %s: %s. Auth label: %s", workflowExecution.ExecutionId, curAuth.OrgId, field.Key, err, curAuth.Label)
setField = false
//fieldLength = 0
break
} else {
continue
}
}
// Remove / at end of urls
// TYPICALLY shouldn't use them.
if field.Key == "url" {
//log.Printf("Value2 (%s): %s", field.Key, string(newValue))
if strings.HasSuffix(string(newValue), "/") {
newValue = []byte(string(newValue)[0 : len(newValue)-1])
}
}
fieldLength += len(newValue)
field.Value = string(newValue)
newFields = append(newFields, field)
}
// There is some Very weird bug that has caused encryption to sometimes be skipped.
// This is a way to discover when this happens properly.
// The problem happens about every 10.000~ decryption, which is still way too much.
// By adding the full total, there should be no problem with this, seeing as lengths are added together
fieldNames := ""
for _, field := range curAuth.Fields {
fieldNames += field.Key + ", "
}
if setField {
curAuth.Fields = newFields
//log.Printf("[DEBUG] Outer decryption (1) debugging for %s. Auth: %s, Fields: %s. Length: %d", curAuth.OrgId, curAuth.Label, fieldNames, fieldLength)
} else {
//log.Printf("[ERROR] Outer decryption (2) debugging for org %s. Auth: '%s'. Fields: %s. Length: %d", curAuth.OrgId, curAuth.Label, fieldNames, fieldLength)
}
} else {
standaloneEnv := os.Getenv("STANDALONE")
if standaloneEnv != "true" {
err := SetWorkflowAppAuthDatastore(ctx, curAuth, curAuth.Id)
if err != nil {
log.Printf("[WARNING] Failed running encryption during execution: %s", err)
}
}
}
}
newParams := []WorkflowAppActionParameter{}
if strings.ToLower(curAuth.Type) == "oauth2-app" {
// Check if they need to be decrypted
// Check if it already has a new token in cache from same auth current execution
setAuth := false
executionAuthKey := fmt.Sprintf("oauth2_%s", curAuth.Id)
//log.Printf("[DEBUG] Looking for cached authkey '%s'", executionAuthKey)
execAuthData, err := GetCache(ctx, executionAuthKey)
if err == nil {
//log.Printf("[DEBUG] Successfully retrieved auth wrapper from cache for %s", executionAuthKey)
cacheData := []byte(execAuthData.([]uint8))
appAuthWrapper := AppAuthenticationStorage{}
err = json.Unmarshal(cacheData, &appAuthWrapper)
if err == nil {
//log.Printf("[DEBUG] Successfully unmarshalled auth wrapper from cache for %s", executionAuthKey)
newParams = action.Parameters
for _, param := range appAuthWrapper.Fields {
if param.Key != "access_token" {
continue
}
newParams = append(newParams, WorkflowAppActionParameter{
Name: param.Key,
Value: param.Value,
})
}
setAuth = true
} else {
log.Printf("[ERROR] Failed unmarshalling auth wrapper from cache for %s: %s", executionAuthKey, err)
}
}
if !setAuth {
for fieldIndex, field := range curAuth.Fields {
parsedKey := fmt.Sprintf("%s_%d_%s_%s", curAuth.OrgId, curAuth.Created, curAuth.Label, field.Key)
decrypted, err := HandleKeyDecryption([]byte(field.Value), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decryption (1) in org %s for %s: %s", curAuth.OrgId, field.Key, err)
if field.Key != "access_key" && field.Key != "access_token" {
//log.Printf("[ERROR] Failed decryption (1) in org %s for %s: %s", curAuth.OrgId, field.Key, err)
}
continue
}
curAuth.Fields[fieldIndex].Value = string(decrypted)
//field.Value = decrypted
}
user := User{
Username: "refresh",
ActiveOrg: OrgMini{
Id: curAuth.OrgId,
},
}
newAuth, err := GetOauth2ApplicationPermissionToken(ctx, user, curAuth)
if err != nil {
log.Printf("[ERROR] Failed running oauth request to refresh oauth2 tokens (2): '%s'. Stopping Oauth2 continuation and sending abort for app. This is NOT critical, but means refreshing access_token failed, and it will stop working in the future.", err)
//workflowExecution.Status = "ABORTED"
//workflowExecution.Result = "Oauth2 failed during start of execution. Please re-authenticate the app."
workflowExecution.NotificationsCreated += 1
workflowExecution.Results = append(workflowExecution.Results, ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: fmt.Sprintf(`{"success": false, "reason": "Failed running oauth2 request to refresh tokens. Are your credentials and URL correct? Contact support@shuffler.io if this persists.", "details": "%s"}`, strings.ReplaceAll(fmt.Sprintf("%s", err), `"`, `\"`)),
StartedAt: workflowExecution.StartedAt,
CompletedAt: workflowExecution.StartedAt,
Status: "SKIPPED",
})
CreateOrgNotification(
ctx,
fmt.Sprintf("Failed to refresh Oauth2 tokens for auth '%s'. Did the credentials change?", curAuth.Label),
fmt.Sprintf("Failed running oauth2 request to refresh oauth2 tokens for app '%s'. Are your credentials and URL correct? Please check backend logs for more details or contact support@shiffler.io for additional help. Details: %#v", curAuth.App.Name, err.Error()),
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, action.ID),
workflowExecution.ExecutionOrg,
true,
"HIGH",
"OAUTH2_REFRESH_FAILURE",
)
// Abort the workflow due to auth being bad
} else {
// Resets the params and overwrites with the relevant fields
curAuth = newAuth
newParams = action.Parameters
for _, param := range newAuth.Fields {
if param.Key != "access_token" {
continue
}
newParams = append(newParams, WorkflowAppActionParameter{
Name: param.Key,
Value: param.Value,
})
}
marshalledAuth, err := json.Marshal(newAuth)
if err == nil {
err = SetCache(ctx, executionAuthKey, marshalledAuth, 1)
if err != nil {
log.Printf("[ERROR] Failed setting cache for %s: %s", executionAuthKey, err)
}
} else {
log.Printf("[ERROR] Failed marshalling auth wrapper for %s: %s", executionAuthKey, err)
}
}
}
} else if strings.ToLower(curAuth.Type) == "oauth2" {
if debug {
log.Printf("[DEBUG][%s] Should replace auth parameters (Oauth2)", workflowExecution.ExecutionId)
}
runRefresh := false
refreshUrl := ""
for _, param := range curAuth.Fields {
if param.Key == "expiration" {
val, err := strconv.Atoi(param.Value)
timeNow := int64(time.Now().Unix())
if err == nil {
//log.Printf("Checking expiration vs timenow: %d %d. Err: %s", timeNow, int64(val)+120, err)
if timeNow >= int64(val)+120 {
runRefresh = true
}
}
continue
}
if param.Key == "refresh_url" {
refreshUrl = param.Value
continue
}
if param.Key != "url" && param.Key != "access_token" {
//if debug {
// log.Printf("[DEBUG][%s] Skipping key %s in auth %s (%s)", workflowExecution.ExecutionId, param.Key, curAuth.Label, curAuth.Id)
//}
continue
}
newParams = append(newParams, WorkflowAppActionParameter{
Name: param.Key,
Value: param.Value,
})
}
// FIXME: Refresh isn't required ALL the time
// but we currently are doing it.
runRefresh = true
if runRefresh {
user := User{
Username: "refresh",
ActiveOrg: OrgMini{
Id: curAuth.OrgId,
},
}
if len(refreshUrl) == 0 {
log.Printf("[ERROR] No Oauth2 request to run, as no refresh url is set!")
} else {
if debug {
log.Printf("[INFO][%s] Running Oauth2 request with URL %s", workflowExecution.ExecutionId, refreshUrl)
}
newAuth, err := RunOauth2Request(ctx, user, curAuth, true)
if err != nil {
log.Printf("[ERROR] Failed running oauth request to refresh oauth2 tokens (1): '%s'. Stopping Oauth2 continuation and sending abort for app. This is NOT critical, but means refreshing access_token failed, and it will stop working in the future.", err)
CreateOrgNotification(
ctx,
fmt.Sprintf("Failed to refresh Oauth2 tokens for app '%s'", curAuth.Label),
fmt.Sprintf("Failed running oauth2 request to refresh oauth2 tokens for app '%s'. Are your credentials and URL correct? Please check backend logs for more details or contact support@shiffler.io for additional help. Details: %#v", curAuth.App.Name, err.Error()),
fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions&node=%s", workflowExecution.Workflow.ID, workflowExecution.ExecutionId, action.ID),
workflowExecution.ExecutionOrg,
true,
"HIGH",
"OAUTH2_REFRESH_FAILURE",
)
// Adding so it can be used to fail the auth naturally with Outlook
authfieldFound := false
for _, field := range curAuth.Fields {
if field.Key == "access_token" {
authfieldFound = true
break
}
}
if !authfieldFound {
newAuth.Fields = append(newAuth.Fields, AuthenticationStore{
Key: "access_token",
Value: "FAILURE_REFRESH",
})
}
// FIXME: There used to be code here to stop the app, but for now we just continue with the old tokens, as it usually works.
}
allAuths[authIndex] = newAuth
// Does the oauth2 replacement
newParams = []WorkflowAppActionParameter{}
for _, param := range newAuth.Fields {
if param.Key != "url" && param.Key != "access_token" {
//log.Printf("Skipping key %s (2)", param.Key)
continue
}
newParams = append(newParams, WorkflowAppActionParameter{
Name: param.Key,
Value: param.Value,
Configuration: true,
})
}
}
}
for _, param := range action.Parameters {
if param.Configuration {
continue
}
newParams = append(newParams, param)
}
} else {
// This may make the system miss fields.
addedParamIndexes := []string{}
for _, param := range action.Parameters {
for paramIndex, authparam := range curAuth.Fields {
if param.Name != authparam.Key {
continue
}
addedParamIndexes = append(addedParamIndexes, fmt.Sprintf("%d", paramIndex))
param.Value = authparam.Value
break
}
newParams = append(newParams, param)
}
for paramIndex, authparam := range curAuth.Fields {
if ArrayContains(addedParamIndexes, fmt.Sprintf("%d", paramIndex)) {
continue
}
newParams = append(newParams, WorkflowAppActionParameter{
Name: authparam.Key,
Value: authparam.Value,
})
}
}
action.Parameters = newParams
return action, workflowExecution
}
func executeAuthgroupSubflow(workflowExecution WorkflowExecution, authgroup AppAuthenticationGroup, apikey string) error {
if len(apikey) == 0 {
log.Printf("[ERROR] No admin API key found to handle subflow execution")
return errors.New("No API key found for subflow execution")
}
log.Printf("[DEBUG] Starting authgroup subflow execution for %s with authgroup %s", workflowExecution.ExecutionId, authgroup.Label)
parsedEnvironment := authgroup.Environment
parsedAuthIds := ""
relevantAuth := map[string]string{}
for _, auth := range authgroup.AppAuths {
if len(auth.App.ID) == 0 {
continue
}
relevantAuth[auth.App.ID] = auth.Id
}
for key, value := range relevantAuth {
parsedAuthIds += fmt.Sprintf("%s=%s;", key, value)
}
parsedAuthIds = strings.TrimRight(parsedAuthIds, ";")
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
resultUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", backendUrl, workflowExecution.Workflow.ID)
// FIXME: Missing source node (?)
//queries := fmt.Sprintf("authgroups=true&source_workflow=authgroups&startnode=%s&source_execution=%s", workflowExecution.Start, workflowExecution.ExecutionId)
urlEncodedLabel := url.QueryEscape(authgroup.Label)
queries := fmt.Sprintf("authgroups=true&authgroup=%s&source_workflow=%s&startnode=%s&source_execution=%s", urlEncodedLabel, workflowExecution.Workflow.Start, workflowExecution.Start, workflowExecution.ExecutionId)
//if action.AuthenticationId == "authgroups" && !subExecutionsDone && workflowExecution.ExecutionSource != "authgroup" {
//sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"]
resultUrl += "?" + queries
log.Printf("\n\n\n[DEBUG][%s] Running subflow execution for workflow %s (%s) with URL %s\n\n", workflowExecution.ExecutionId, workflowExecution.Workflow.Name, workflowExecution.Workflow.ID, resultUrl)
preparedRuntime := ExecutionRequest{
Priority: 10,
ExecutionSource: "authgroups",
Start: workflowExecution.Start,
WorkflowId: workflowExecution.Workflow.ID,
Environments: []string{parsedEnvironment},
ExecutionArgument: workflowExecution.ExecutionArgument,
Authgroup: authgroup.Label,
}
topClient := GetExternalClient(backendUrl)
data, err := json.Marshal(preparedRuntime)
if err != nil {
log.Printf("[WARNING] Failed parent init marshal: %s", err)
return err
}
req, err := http.NewRequest(
"POST",
resultUrl,
bytes.NewBuffer([]byte(data)),
)
if len(apikey) == 0 {
return errors.New("No API key found for subflow execution")
}
if strings.HasPrefix(apikey, "Bearer ") && len(apikey) > 7 {
apikey = apikey[7:]
}
req.Header.Set("Org-Identifier", workflowExecution.ExecutionOrg)
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apikey))
req.Header.Set("appauth", parsedAuthIds)
req.Header.Set("environment", parsedEnvironment)
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Failed making authgroup subflow request (1): %s. Is URL valid: %s", err, resultUrl)
return err
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading body response from authgroup exec: %s", err)
return err
}
log.Printf("AUTHGROUP RUN BODY (%d): %s", newresp.StatusCode, string(body))
if newresp.StatusCode != 200 {
log.Printf("[ERROR] Bad statuscode running authgroup execution (2) with URL %s: %d, %s", resultUrl, newresp.StatusCode, string(body))
return errors.New(fmt.Sprintf("Bad statuscode: %d", newresp.StatusCode))
}
return nil
}
// Extra validation sample to be used for workflow executions based on parent workflow instead of users' auth
// Check if the execution data has correct info in it! Happens based on subflows.
// 1. Parent workflow contains this workflow ID in the source trigger?
// 2. Parent workflow's owner is same org?
// 3. Parent execution auth is correct
func RunExecuteAccessValidation(request *http.Request, workflow *Workflow) (bool, string) {
if debug {
log.Printf("[DEBUG] Inside execute validation for workflow %s (%s)! Request method: %s. Queries: %#v", workflow.Name, workflow.ID, request.Method, request.URL.Query())
}
//if request.Method == "POST" {
ctx := GetContext(request)
workflowExecution := &WorkflowExecution{}
sourceExecution, sourceExecutionOk := request.URL.Query()["source_execution"]
if !sourceExecutionOk {
sourceExecution, sourceExecutionOk = request.URL.Query()["reference_execution"]
if !sourceExecutionOk {
sourceExecution, sourceExecutionOk = request.URL.Query()["execution_id"]
if !sourceExecutionOk {
log.Printf("[AUDIT] No source execution found for workflow execution of %s (%s). Queries: %#v. Bad auth.", workflow.Name, workflow.ID, request.URL.Query())
return false, ""
}
}
}
if debug == true {
log.Printf("[DEBUG] Source execution in exec auth: %s", sourceExecution[0])
}
if sourceExecutionOk && len(sourceExecution) > 0 {
//log.Printf("[DEBUG] Got source exec %s", sourceExecution)
newExec, err := GetWorkflowExecution(ctx, sourceExecution[0])
if err != nil {
if debug {
log.Printf("[DEBUG] Failed getting source_execution in test validation based on '%s'", sourceExecution[0])
}
return false, ""
} else {
workflowExecution = newExec
}
}
if workflowExecution.ExecutionId == "" {
if debug {
log.Printf("[DEBUG] No execution ID found. Bad auth. Source Exec: %s", sourceExecution[0])
}
return false, ""
}
sourceAuth, sourceAuthOk := request.URL.Query()["source_auth"]
if !sourceAuthOk {
sourceAuth, sourceAuthOk = request.URL.Query()["authorization"]
if !sourceAuthOk {
if debug {
log.Printf("[DEBUG] No source auth found during execution of %s. Bad auth.", workflowExecution.ExecutionId)
}
return false, ""
}
}
if sourceAuthOk {
if sourceAuth[0] != workflowExecution.Authorization {
log.Printf("[AUDIT] Bad authorization for workflowexecution defined.")
return false, ""
}
// Check if workflow is in waiting stage
// If it is, accept from this point already, as it's a user input action
if workflowExecution.Status == "WAITING" {
return true, ""
}
}
if debug {
log.Printf("[DEBUG] Source auth: %s", sourceAuth[0])
}
// Need to verify the workflow, and whether it SHOULD have access to execute it.
sourceWorkflow, sourceWorkflowOk := request.URL.Query()["source_workflow"]
if sourceWorkflowOk {
_ = sourceWorkflow
// Do more checks here? Does it matter?
} else {
if len(workflowExecution.Workflow.ID) > 0 {
log.Printf("[AUDIT][%s] Got source workflow in subflow execution. Continuing.", workflowExecution.ExecutionId)
} else {
log.Printf("[AUDIT] Did NOT get source workflow in subflow execution. Failing out.")
return false, ""
}
}
//if workflow.OrgId != workflowExecution.Workflow.OrgId || workflow.ExecutingOrg.Id != workflowExecution.Workflow.ExecutingOrg.Id || workflow.OrgId == "" {
//if len(workflow.OrgId) > 0 && workflow.OrgId != workflowExecution.Workflow.OrgId {
if workflow.OrgId == "" || workflow.OrgId != workflowExecution.Workflow.OrgId {
log.Printf("[ERROR][%s] Bad org ID in workflowexecution subflow run. Required: %s vs %s", workflowExecution.ExecutionId, workflow.OrgId, workflowExecution.Workflow.OrgId)
return false, ""
}
return true, workflowExecution.ExecutionOrg
}
// Significantly slowed down everything. Just returning for now.
func findReferenceAppDocs(ctx context.Context, allApps []WorkflowApp) []WorkflowApp {
newApps := []WorkflowApp{}
// Skipping this for now as it makes things slow
return allApps
for _, app := range allApps {
if len(app.ReferenceInfo.DocumentationUrl) > 0 && strings.HasPrefix(app.ReferenceInfo.DocumentationUrl, "https://raw.githubusercontent.com/Shuffle") && strings.Contains(app.ReferenceInfo.DocumentationUrl, ".md") {
// Should find documentation from the github (only if github?) and add it to app.Documentation before caching
//log.Printf("DOCS: %s", app.ReferenceInfo.DocumentationUrl)
documentationData, err := DownloadFromUrl(ctx, app.ReferenceInfo.DocumentationUrl)
if err != nil {
log.Printf("[ERROR] Failed getting data: %s", err)
} else {
app.Documentation = string(documentationData)
}
}
//if app.Documentation == "" && strings.ToLower(app.Name) == "discord" {
if app.Documentation == "" {
referenceUrl := ""
if app.Generated {
//log.Printf("[DEBUG] Should look in the OpenAPI folder")
baseUrl := "https://raw.githubusercontent.com/Shuffle/openapi-apps/master/docs"
newName := strings.ToLower(strings.Replace(strings.Replace(app.Name, " ", "_", -1), "-", "_", -1))
referenceUrl = fmt.Sprintf("%s/%s.md", baseUrl, newName)
} else {
//log.Printf("[DEBUG] Should look in the Python-apps folder")
}
if len(referenceUrl) > 0 {
//log.Printf("REF: %s", referenceUrl)
documentationData, err := DownloadFromUrl(ctx, referenceUrl)
if err != nil {
log.Printf("[ERROR] Failed getting documentation data for app %s: %s", app.Name, err)
} else {
//log.Printf("[INFO] Added documentation from github for %s", app.Name)
app.Documentation = string(documentationData)
}
}
}
newApps = append(newApps, app)
}
return newApps
}
func CheckGitProxy(cloneOptions *git.CloneOptions) *git.CloneOptions {
if os.Getenv("HTTP_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) {
cloneOptions.ProxyOptions = transport.ProxyOptions{
URL: os.Getenv("HTTP_PROXY"),
}
}
if os.Getenv("HTTPS_PROXY") != "" && !isGitNoProxy(cloneOptions.URL) {
cloneOptions.ProxyOptions = transport.ProxyOptions{
URL: os.Getenv("HTTPS_PROXY"),
}
}
return cloneOptions
}
func isGitNoProxy(rawURL string) bool {
noProxy := os.Getenv("NO_PROXY")
if noProxy == "" {
return false
}
if noProxy == "*" {
return true
}
noProxyList := strings.Split(noProxy, ",")
parsedURL, err := url.Parse(rawURL)
if err != nil {
return false
}
host := parsedURL.Hostname()
for _, value := range noProxyList {
value = strings.TrimSpace(value)
if host == value {
return true
}
if strings.HasPrefix(value, "*.") && strings.HasSuffix(host, value[2:]) {
return true
}
}
return false
}
func loadGithubWorkflows(url, username, password, userId, branch, orgId string) error {
fs := memfs.New()
// Extract path parameter from Azure DevOps URLs
targetPath := ""
specificFile := ""
if strings.Contains(url, "dev.azure.com") {
if strings.Contains(url, "?path=") {
parts := strings.Split(url, "?path=")
if len(parts) == 2 {
targetPath = parts[1]
decodedPath, err := neturl.QueryUnescape(targetPath)
if err == nil {
targetPath = decodedPath
}
targetPath = strings.TrimPrefix(targetPath, "/")
if strings.HasSuffix(strings.ToLower(targetPath), ".json") {
specificFile = path.Base(targetPath)
targetPath = path.Dir(targetPath)
if targetPath == "." {
targetPath = ""
}
}
log.Printf("[INFO] Extracted target path: '%s', specific file: '%s'", targetPath, specificFile)
}
// Clean the URL
url = parts[0]
}
}
// Handle GitHub-specific file URLs
if strings.Contains(url, "github.com") && (strings.Contains(url, "/blob/") || strings.Contains(url, "/tree/")) {
separator := "/blob/"
if strings.Contains(url, "/tree/") {
separator = "/tree/"
}
parts := strings.SplitN(url, separator, 2)
if len(parts) == 2 {
baseURL := parts[0]
remainder := parts[1]
pathParts := strings.SplitN(remainder, "/", 2)
if len(pathParts) >= 1 {
// If branch wasn't already specified, use the one from URL
if len(branch) == 0 || branch == "main" || branch == "master" {
branch = pathParts[0]
}
if len(pathParts) == 2 {
targetPath = pathParts[1]
// Check if it's a specific file
if strings.HasSuffix(strings.ToLower(targetPath), ".json") {
specificFile = path.Base(targetPath)
targetPath = path.Dir(targetPath)
if targetPath == "." {
targetPath = ""
}
}
}
}
// Convert to proper git URL
url = baseURL + ".git"
log.Printf("[INFO] Converted GitHub URL. Repo: %s, Branch: %s, Path: '%s', File: '%s'", url, branch, targetPath, specificFile)
}
}
log.Printf("Starting load of %s with branch %s", url, branch)
cloneOptions := &git.CloneOptions{
URL: url,
}
if len(username) > 0 && len(password) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: username,
Password: password,
}
} else {
org, err := GetOrg(context.Background(), orgId)
if err != nil {
log.Printf("Failed getting org %s: %s", orgId, err)
return err
}
// check if org has git credentials
if len(org.Defaults.WorkflowUploadUsername) > 0 && len(org.Defaults.WorkflowUploadToken) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername,
Password: org.Defaults.WorkflowUploadToken,
}
}
}
// main is the new master
if len(branch) > 0 && branch != "main" && branch != "master" {
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
}
cloneOptions = CheckGitProxy(cloneOptions)
// Azure DevOps requires special capability handling
isAzureDevOps := strings.Contains(url, "dev.azure.com")
if isAzureDevOps {
transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}
}
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("[INFO] Failed loading repo %s into memory (github workflows): %s", url, err)
return err
}
// Navigate to the target directory if specified
searchPath := "/"
if targetPath != "" {
searchPath = "/" + targetPath
log.Printf("[INFO] Navigating to target path: %s", searchPath)
}
dir, err := fs.ReadDir(searchPath)
if err != nil {
log.Printf("[ERROR] Failed reading folder '%s': %s", searchPath, err)
// Try without leading slash
searchPath = strings.TrimPrefix(searchPath, "/")
if searchPath == "" {
searchPath = "."
}
dir, err = fs.ReadDir(searchPath)
if err != nil {
log.Printf("[ERROR] Failed reading folder '%s' (retry): %s", searchPath, err)
return err
}
}
_ = r
// Prepare the extra parameter for the iteration function
// Always use forward slashes for git filesystem paths
extraPath := ""
if targetPath != "" {
extraPath = targetPath
// Ensure forward slashes
extraPath = strings.ReplaceAll(extraPath, "\\", "/")
if !strings.HasSuffix(extraPath, "/") {
extraPath += "/"
}
}
log.Printf("[INFO] Starting workflow folder iteration in path '%s' with specific file: '%s'", searchPath, specificFile)
iterateWorkflowGithubFolders(fs, dir, extraPath, specificFile, userId, orgId)
return nil
}
// listGithubWorkflowsInfo clones a git repo and returns metadata for each workflow JSON found.
// It also checks whether each workflow ID already exists in the given org.
func listGithubWorkflowsInfo(url, username, password, branch, orgId string) ([]RemoteWorkflowInfo, error) {
fs := memfs.New()
targetPath := ""
specificFile := ""
if strings.Contains(url, "dev.azure.com") {
if strings.Contains(url, "?path=") {
parts := strings.Split(url, "?path=")
if len(parts) == 2 {
targetPath = parts[1]
decodedPath, err := neturl.QueryUnescape(targetPath)
if err == nil {
targetPath = decodedPath
}
targetPath = strings.TrimPrefix(targetPath, "/")
if strings.HasSuffix(strings.ToLower(targetPath), ".json") {
specificFile = path.Base(targetPath)
targetPath = path.Dir(targetPath)
if targetPath == "." {
targetPath = ""
}
}
url = parts[0]
}
}
}
if strings.Contains(url, "github.com") && (strings.Contains(url, "/blob/") || strings.Contains(url, "/tree/")) {
separator := "/blob/"
if strings.Contains(url, "/tree/") {
separator = "/tree/"
}
parts := strings.SplitN(url, separator, 2)
if len(parts) == 2 {
baseURL := parts[0]
remainder := parts[1]
pathParts := strings.SplitN(remainder, "/", 2)
if len(pathParts) >= 1 {
if len(branch) == 0 || branch == "main" || branch == "master" {
branch = pathParts[0]
}
if len(pathParts) == 2 {
targetPath = pathParts[1]
if strings.HasSuffix(strings.ToLower(targetPath), ".json") {
specificFile = path.Base(targetPath)
targetPath = path.Dir(targetPath)
if targetPath == "." {
targetPath = ""
}
}
}
}
url = baseURL + ".git"
}
}
cloneOptions := &git.CloneOptions{URL: url}
if len(username) > 0 && len(password) > 0 {
cloneOptions.Auth = &http2.BasicAuth{Username: username, Password: password}
} else {
org, err := GetOrg(context.Background(), orgId)
if err == nil && len(org.Defaults.WorkflowUploadUsername) > 0 && len(org.Defaults.WorkflowUploadToken) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername,
Password: org.Defaults.WorkflowUploadToken,
}
}
}
if len(branch) > 0 && branch != "main" && branch != "master" {
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
}
cloneOptions = CheckGitProxy(cloneOptions)
isAzureDevOps := strings.Contains(url, "dev.azure.com")
if isAzureDevOps {
transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}
}
storer := memory.NewStorage()
_, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("[INFO] Failed cloning repo %s for list: %s", url, err)
return nil, err
}
searchPath := "/"
if targetPath != "" {
searchPath = "/" + targetPath
}
dir, err := fs.ReadDir(searchPath)
if err != nil {
searchPath = strings.TrimPrefix(searchPath, "/")
if searchPath == "" {
searchPath = "."
}
dir, err = fs.ReadDir(searchPath)
if err != nil {
return nil, err
}
}
extraPath := ""
if targetPath != "" {
extraPath = strings.ReplaceAll(targetPath, "\\", "/")
if !strings.HasSuffix(extraPath, "/") {
extraPath += "/"
}
}
// Collect all remote workflow infos.
// Must be an initialized (non-nil) slice so it serializes as [] not null.
remoteInfos := make([]RemoteWorkflowInfo, 0)
collectWorkflowInfos(fs, dir, extraPath, specificFile, &remoteInfos)
// Deduplicate by workflow ID, keeping the entry with the most recent UpdatedAt
seenIds := make(map[string]int) // id -> index in dedupInfos
dedupInfos := make([]RemoteWorkflowInfo, 0, len(remoteInfos))
for _, info := range remoteInfos {
if existingIdx, seen := seenIds[info.ID]; seen {
if info.UpdatedAt > dedupInfos[existingIdx].UpdatedAt {
dedupInfos[existingIdx] = info
}
} else {
seenIds[info.ID] = len(dedupInfos)
dedupInfos = append(dedupInfos, info)
}
}
remoteInfos = dedupInfos
// Check which workflows already exist in this org
ctx := context.Background()
for i, info := range remoteInfos {
existing, err := GetWorkflow(ctx, info.ID)
if err == nil && existing != nil && existing.OrgId == orgId {
remoteInfos[i].ExistsInOrg = true
remoteInfos[i].OrgWorkflowId = existing.ID
}
}
log.Printf("[INFO] listGithubWorkflowsInfo: found %d workflows in remote repo", len(remoteInfos))
return remoteInfos, nil
}
// collectWorkflowInfos recursively walks the billy filesystem and collects RemoteWorkflowInfo for each .json file.
func collectWorkflowInfos(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, infos *[]RemoteWorkflowInfo) {
for _, file := range dir {
filename := file.Name()
if len(onlyname) > 0 && filename != onlyname {
continue
}
fullPath := fmt.Sprintf("%s%s", extra, filename)
fi, err := fs.Stat(fullPath)
if err != nil {
continue
}
switch mode := fi.Mode(); {
case mode.IsDir():
tmpExtra := fmt.Sprintf("%s%s/", extra, fi.Name())
subDir, err := fs.ReadDir(tmpExtra)
if err != nil {
continue
}
collectWorkflowInfos(fs, subDir, tmpExtra, "", infos)
case mode.IsRegular():
if !strings.HasSuffix(strings.ToLower(filename), ".json") {
continue
}
filePath := fmt.Sprintf("%s%s", extra, fi.Name())
f, err := fs.Open(filePath)
if err != nil {
continue
}
data, err := ioutil.ReadAll(f)
if err != nil {
continue
}
var wf Workflow
if err := json.Unmarshal(data, &wf); err != nil || wf.ID == "" || wf.Name == "" {
continue
}
folderName := strings.TrimSuffix(extra, "/")
if idx := strings.LastIndex(folderName, "/"); idx >= 0 {
folderName = folderName[idx+1:]
}
*infos = append(*infos, RemoteWorkflowInfo{
ID: wf.ID,
Name: wf.Name,
FolderName: folderName,
UpdatedAt: wf.Edited,
FilePath: filePath,
})
}
}
}
// importSingleRemoteWorkflow imports or syncs a single workflow from a git repo by its original ID.
// If syncToId is non-empty, the existing workflow with that ID is updated (synced).
// Otherwise a new workflow is created (imported).
func importSingleRemoteWorkflow(url, username, password, branch, originalWorkflowId, syncToId, userId, orgId string) error {
fs := memfs.New()
targetPath := ""
if strings.Contains(url, "dev.azure.com") {
if strings.Contains(url, "?path=") {
parts := strings.Split(url, "?path=")
if len(parts) == 2 {
targetPath = parts[1]
if d, err := neturl.QueryUnescape(targetPath); err == nil {
targetPath = d
}
targetPath = strings.TrimPrefix(targetPath, "/")
if strings.HasSuffix(strings.ToLower(targetPath), ".json") {
targetPath = path.Dir(targetPath)
if targetPath == "." {
targetPath = ""
}
}
url = parts[0]
}
}
}
if strings.Contains(url, "github.com") && (strings.Contains(url, "/blob/") || strings.Contains(url, "/tree/")) {
separator := "/blob/"
if strings.Contains(url, "/tree/") {
separator = "/tree/"
}
parts := strings.SplitN(url, separator, 2)
if len(parts) == 2 {
baseURL := parts[0]
remainder := parts[1]
pathParts := strings.SplitN(remainder, "/", 2)
if len(pathParts) >= 1 {
if len(branch) == 0 || branch == "main" || branch == "master" {
branch = pathParts[0]
}
if len(pathParts) == 2 {
rp := pathParts[1]
if strings.HasSuffix(strings.ToLower(rp), ".json") {
rp = path.Dir(rp)
if rp == "." {
rp = ""
}
}
targetPath = rp
}
}
url = baseURL + ".git"
}
}
cloneOptions := &git.CloneOptions{URL: url}
if len(username) > 0 && len(password) > 0 {
cloneOptions.Auth = &http2.BasicAuth{Username: username, Password: password}
} else {
org, err := GetOrg(context.Background(), orgId)
if err == nil && len(org.Defaults.WorkflowUploadUsername) > 0 && len(org.Defaults.WorkflowUploadToken) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername,
Password: org.Defaults.WorkflowUploadToken,
}
}
}
if len(branch) > 0 && branch != "main" && branch != "master" {
cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
}
cloneOptions = CheckGitProxy(cloneOptions)
isAzureDevOps := strings.Contains(url, "dev.azure.com")
if isAzureDevOps {
transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}
}
storer := memory.NewStorage()
_, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
return err
}
searchPath := "/"
if targetPath != "" {
searchPath = "/" + targetPath
}
dir, err := fs.ReadDir(searchPath)
if err != nil {
searchPath = strings.TrimPrefix(searchPath, "/")
if searchPath == "" {
searchPath = "."
}
dir, err = fs.ReadDir(searchPath)
if err != nil {
return err
}
}
extraPath := ""
if targetPath != "" {
extraPath = strings.ReplaceAll(targetPath, "\\", "/")
if !strings.HasSuffix(extraPath, "/") {
extraPath += "/"
}
}
return findAndProcessSingleWorkflow(fs, dir, extraPath, originalWorkflowId, syncToId, userId, orgId)
}
// findAndProcessSingleWorkflow walks the FS looking for the workflow with the given original ID.
func findAndProcessSingleWorkflow(fs billy.Filesystem, dir []os.FileInfo, extra, originalWorkflowId, syncToId, userId, orgId string) error {
for _, file := range dir {
filename := file.Name()
fullPath := fmt.Sprintf("%s%s", extra, filename)
fi, err := fs.Stat(fullPath)
if err != nil {
continue
}
switch mode := fi.Mode(); {
case mode.IsDir():
tmpExtra := fmt.Sprintf("%s%s/", extra, fi.Name())
subDir, err := fs.ReadDir(tmpExtra)
if err != nil {
continue
}
if err := findAndProcessSingleWorkflow(fs, subDir, tmpExtra, originalWorkflowId, syncToId, userId, orgId); err == nil {
return nil // found and processed
}
case mode.IsRegular():
if !strings.HasSuffix(strings.ToLower(filename), ".json") {
continue
}
filePath := fmt.Sprintf("%s%s", extra, fi.Name())
f, err := fs.Open(filePath)
if err != nil {
continue
}
data, err := ioutil.ReadAll(f)
if err != nil {
continue
}
var wf Workflow
if err := json.Unmarshal(data, &wf); err != nil {
continue
}
if wf.ID != originalWorkflowId {
continue
}
ctx := context.Background()
if syncToId != "" {
// Sync: update the existing org workflow with the remote content
existing, err := GetWorkflow(ctx, syncToId)
if err != nil || existing == nil {
return fmt.Errorf("could not find org workflow %s to sync: %v", syncToId, err)
}
// Preserve org ownership, update content
wf.ID = existing.ID
wf.Owner = existing.Owner
wf.OrgId = existing.OrgId
wf.ExecutingOrg = existing.ExecutingOrg
wf.Org = existing.Org
wf.IsValid = existing.IsValid
log.Printf("[INFO] Syncing remote workflow '%s' into org workflow %s", wf.Name, syncToId)
return SetWorkflow(ctx, wf, wf.ID)
}
// Import: preserve the original workflow ID embedded in the repo JSON file.
// Do NOT generate a new UUID — the ID from the file is the canonical identifier.
wf.Owner = userId
wf.OrgId = orgId
wf.ExecutingOrg = OrgMini{Id: orgId}
wf.Org = append(wf.Org, OrgMini{Id: orgId})
wf.IsValid = false
wf.Errors = []string{"Imported, not locally saved. Save before using."}
// Restore app images
workflowapps, err := GetAllWorkflowApps(ctx, 1000, 0)
if err == nil {
for actionIndex, action := range wf.Actions {
if action.AppID == "" {
continue
}
for _, app := range workflowapps {
if (app.ID == action.AppID || app.Name == action.AppName) && app.AppVersion == action.AppVersion {
wf.Actions[actionIndex].LargeImage = app.LargeImage
wf.Actions[actionIndex].SmallImage = app.LargeImage
break
}
}
}
}
log.Printf("[INFO] Importing remote workflow '%s' as new workflow %s", wf.Name, wf.ID)
return SetWorkflow(ctx, wf, wf.ID)
}
}
return fmt.Errorf("workflow with id %s not found in remote repo", originalWorkflowId)
}
func LoadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just need to be logged in
// FIXME - should have some permissions?
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in load apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("Wrong user (%s) when downloading from github", user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Downloading remotely requires admin"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Field1 & 2 can be a lot of things..
type tmpStruct struct {
URL string `json:"url"`
Username string `json:"username"`
Password string `json:"password"`
Branch string `json:"branch"`
ListOnly bool `json:"list_only"`
OriginalWorkflowId string `json:"original_workflow_id"`
SyncToId string `json:"sync_to_id"`
}
var tmpBody tmpStruct
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("Error with unmarshal tmpBody: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Mode 1: list_only – return metadata for all workflows in the repo without importing
if tmpBody.ListOnly {
infos, err := listGithubWorkflowsInfo(tmpBody.URL, tmpBody.Username, tmpBody.Password, tmpBody.Branch, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] listGithubWorkflowsInfo failed: %s", err)
type errResp struct {
Success bool `json:"success"`
Reason string `json:"reason"`
}
out, _ := json.Marshal(errResp{Success: false, Reason: err.Error()})
resp.WriteHeader(400)
resp.Write(out)
return
}
type listResp struct {
Success bool `json:"success"`
Workflows []RemoteWorkflowInfo `json:"workflows"`
}
// Filter out all workflow with name "Ops Dashboard Workflow" skip this workflows are those are health workflows
filteredInfos := []RemoteWorkflowInfo{}
for _, info := range infos {
if info.Name == "Ops Dashboard Workflow" {
continue
}
filteredInfos = append(filteredInfos, info)
}
out, _ := json.Marshal(listResp{Success: true, Workflows: filteredInfos})
resp.WriteHeader(200)
resp.Write(out)
return
}
// Mode 2: single workflow import or sync
if tmpBody.OriginalWorkflowId != "" {
err = importSingleRemoteWorkflow(tmpBody.URL, tmpBody.Username, tmpBody.Password, tmpBody.Branch, tmpBody.OriginalWorkflowId, tmpBody.SyncToId, user.Id, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] importSingleRemoteWorkflow failed: %s", err)
type errResp struct {
Success bool `json:"success"`
Reason string `json:"reason"`
}
out, _ := json.Marshal(errResp{Success: false, Reason: err.Error()})
resp.WriteHeader(400)
resp.Write(out)
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
return
}
// Mode 3: original bulk import
err = loadGithubWorkflows(tmpBody.URL, tmpBody.Username, tmpBody.Password, user.Id, tmpBody.Branch, user.ActiveOrg.Id)
if err != nil {
log.Printf("Failed to update workflows: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
// Onlyname is used to
func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname, userId, orgId string) error {
var err error
secondsOffset := 0
// sort file names
filenames := []string{}
for _, file := range dir {
filename := file.Name()
filenames = append(filenames, filename)
}
sort.Strings(filenames)
// iterate through sorted filenames
for _, filename := range filenames {
secondsOffset -= 10
if len(onlyname) > 0 && filename != onlyname {
continue
}
// Construct full path for stat
fullPath := fmt.Sprintf("%s%s", extra, filename)
file, err := fs.Stat(fullPath)
if err != nil {
log.Printf("[DEBUG] Failed to stat file '%s': %s", fullPath, err)
continue
}
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
dir, err := fs.ReadDir(tmpExtra)
if err != nil {
log.Printf("Failed to read dir: %s", err)
continue
}
// Go routine? Hmm, this can be super quick I guess
err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "", userId, orgId)
if err != nil {
continue
}
case mode.IsRegular():
// Check the file
if strings.HasSuffix(filename, ".json") {
path := fmt.Sprintf("%s%s", extra, file.Name())
fileReader, err := fs.Open(path)
if err != nil {
log.Printf("Error reading file: %s", err)
continue
}
readFile, err := ioutil.ReadAll(fileReader)
if err != nil {
log.Printf("Error reading file: %s", err)
continue
}
var workflow Workflow
err = json.Unmarshal(readFile, &workflow)
if err != nil {
continue
}
// rewrite owner to user who imports now
if userId != "" {
workflow.Owner = userId
}
// Preserve the original workflow ID from the repo JSON file.
// Do NOT generate a new UUID — the ID from the file is the canonical identifier.
workflow.OrgId = orgId
workflow.ExecutingOrg = OrgMini{
Id: orgId,
}
workflow.Org = append(workflow.Org, OrgMini{
Id: orgId,
})
workflow.IsValid = false
workflow.Errors = []string{"Imported, not locally saved. Save before using."}
// Restore app and trigger images
ctx := context.Background()
// Restore app images from app definitions
if len(workflow.Actions) > 0 {
workflowapps, err := GetAllWorkflowApps(ctx, 1000, 0)
if err == nil && len(workflowapps) > 0 {
for actionIndex, action := range workflow.Actions {
if action.AppID == "" {
continue
}
// Find matching app
for _, app := range workflowapps {
if app.ID == action.AppID && app.AppVersion == action.AppVersion {
// Restore app images
workflow.Actions[actionIndex].LargeImage = app.LargeImage
workflow.Actions[actionIndex].SmallImage = ""
if len(app.LargeImage) > 0 {
workflow.Actions[actionIndex].SmallImage = app.LargeImage
}
break
} else if app.Name == action.AppName && app.AppVersion == action.AppVersion {
// Fallback: match by name and version
workflow.Actions[actionIndex].LargeImage = app.LargeImage
workflow.Actions[actionIndex].SmallImage = ""
if len(app.LargeImage) > 0 {
workflow.Actions[actionIndex].SmallImage = app.LargeImage
}
break
}
}
}
}
}
/*
// Find existing similar ones
q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id).Filter("name", workflow.name)
var workflows []Workflow
_, err = dbclient.GetAll(ctx, q, &workflows)
if err == nil {
log.Printf("Failed getting workflows for user %s: %s", user.Username, err)
if len(workflows) == 0 {
resp.WriteHeader(200)
resp.Write([]byte("[]"))
return
}
}
*/
log.Printf("Import workflow from file: %s", filename)
err = SetWorkflow(ctx, workflow, workflow.ID, secondsOffset)
if err != nil {
log.Printf("Failed setting (download) workflow: %s", err)
continue
}
log.Printf("Uploaded workflow %s for user %s and org %s!", filename, userId, orgId)
}
}
}
return err
}
func EchoOpenapiData(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[DEBUG] Api authentication failed in download Yaml echo: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to echo OpenAPI data: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Bodyreader err: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
newbody := string(body)
newbody = strings.TrimSpace(newbody)
if strings.HasPrefix(newbody, "\"") {
newbody = newbody[1:len(newbody)]
}
if strings.HasSuffix(newbody, "\"") {
newbody = newbody[0 : len(newbody)-1]
}
// Rewrite to download proper one from Github even without raw path
if strings.Contains(newbody, "https://github.com/") {
// https://github.com/AdguardTeam/AdGuardHome/blob/master/openapi/openapi.yaml
// https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/openapi/openapi.yaml
// https://raw.githubusercontent.com/AdguardTeam/AdGuardHome/master/openapi/openapi.yaml
urlsplit := strings.Split(newbody, "/")
if len(urlsplit) > 6 {
log.Printf("[DEBUG] Rewriting github URL %s.", newbody)
ghuser := urlsplit[3]
repo := urlsplit[4]
branch := urlsplit[6]
path := strings.Join(urlsplit[7:len(urlsplit)], "/")
newbody = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", ghuser, repo, branch, path)
}
}
log.Printf("[DEBUG] Downloading content from %s", newbody)
req, err := http.NewRequest(
"GET",
newbody,
nil,
)
if err != nil {
log.Printf("[ERROR] Requestbuilder err: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed building request"}`))
return
}
httpClient := &http.Client{}
newresp, err := httpClient.Do(req)
if err != nil {
log.Printf("[ERROR] Grabbing error: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making remote request to get the data"}`)))
return
}
defer newresp.Body.Close()
urlbody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] URLbody error: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"}`)))
return
}
if newresp.StatusCode >= 400 {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, urlbody)))
return
}
resp.WriteHeader(200)
resp.Write(urlbody)
}
func GetFrameworkConfiguration(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[DEBUG] Api authentication failed in get detection framework: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Error getting org %s for user %s (%s): %s", user.ActiveOrg.Id, user.Username, user.Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
newjson, err := json.Marshal(org.SecurityFramework)
if err != nil {
log.Printf("[ERROR] Failed marshal in get security framework: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking framework. Contact us to get it fixed."}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func SetFrameworkConfiguration(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[DEBUG] Api authentication failed in set detection framework: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to set detection framework: %s (%s)", user.Username, user.Id)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type parsedValue struct {
Type string `json:"type"`
Name string `json:"name"`
ID string `json:"id"`
LargeImage string `json:"large_image"`
Description string `json:"description"`
}
var value parsedValue
err = json.Unmarshal(body, &value)
if err != nil {
log.Printf("[WARNING] Error with unmarshal tmpBody in frameworkconfig: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Error getting org in set framework: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
app := &WorkflowApp{
Name: "",
Description: "",
ID: "",
LargeImage: "",
}
// System for replacing an app if it's not defined
if value.ID != "remove" {
app, err = GetApp(ctx, value.ID, user, false)
if err != nil {
if project.Environment == "cloud" {
log.Printf("[ERROR] Error getting app '%s' in set framework: %s", value.ID, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
} else {
// Forwarded from Algolia in the frontend
app.Name = value.Name
app.ID = value.Name
app.Description = value.Description
app.LargeImage = value.LargeImage
}
}
if project.Environment == "cloud" && !app.Sharing && app.Public {
log.Printf("[WARNING] Error setting app %s for org %s as it's not public.", value.ID, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// Trim and lower
value.Type = strings.ToLower(strings.TrimSpace(value.Type))
if value.Type == "email" || value.Type == "comms" {
value.Type = "communication"
}
if value.Type == "eradication" {
value.Type = "edr"
}
if value.Type == "edr & av" {
value.Type = "edr"
}
appPriority := Priority{}
prioIndex := -1
for i, priority := range org.Priorities {
if priority.Type == "apps" && strings.Contains(strings.ToLower(priority.Name), value.Type) && strings.Contains(priority.Name, "/8)") {
appPriority = priority
prioIndex = i
break
}
}
log.Printf("[INFO] Found app priority (%s) with name '%s'", value.Type, appPriority.Name)
if prioIndex >= 0 && strings.Contains(strings.ToLower(appPriority.Name), value.Type) {
if value.ID == "remove" {
org.Priorities[prioIndex].Active = true
} else {
org.Priorities[prioIndex].Active = false
}
}
// 1. Check if the app exists and the user has access to it. If public/sharing ->
if value.Type == "siem" {
org.SecurityFramework.SIEM.Name = app.Name
org.SecurityFramework.SIEM.Description = app.Description
org.SecurityFramework.SIEM.ID = app.ID
org.SecurityFramework.SIEM.LargeImage = app.LargeImage
} else if value.Type == "network" {
org.SecurityFramework.Network.Name = app.Name
org.SecurityFramework.Network.Description = app.Description
org.SecurityFramework.Network.ID = app.ID
org.SecurityFramework.Network.LargeImage = app.LargeImage
} else if value.Type == "edr" {
org.SecurityFramework.EDR.Name = app.Name
org.SecurityFramework.EDR.Description = app.Description
org.SecurityFramework.EDR.ID = app.ID
org.SecurityFramework.EDR.LargeImage = app.LargeImage
} else if value.Type == "cases" {
org.SecurityFramework.Cases.Name = app.Name
org.SecurityFramework.Cases.Description = app.Description
org.SecurityFramework.Cases.ID = app.ID
org.SecurityFramework.Cases.LargeImage = app.LargeImage
} else if value.Type == "iam" {
org.SecurityFramework.IAM.Name = app.Name
org.SecurityFramework.IAM.Description = app.Description
org.SecurityFramework.IAM.ID = app.ID
org.SecurityFramework.IAM.LargeImage = app.LargeImage
} else if value.Type == "assets" {
org.SecurityFramework.Assets.Name = app.Name
org.SecurityFramework.Assets.Description = app.Description
org.SecurityFramework.Assets.ID = app.ID
org.SecurityFramework.Assets.LargeImage = app.LargeImage
} else if value.Type == "intel" {
org.SecurityFramework.Intel.Name = app.Name
org.SecurityFramework.Intel.Description = app.Description
org.SecurityFramework.Intel.ID = app.ID
org.SecurityFramework.Intel.LargeImage = app.LargeImage
} else if value.Type == "communication" {
org.SecurityFramework.Communication.Name = app.Name
org.SecurityFramework.Communication.Description = app.Description
org.SecurityFramework.Communication.ID = app.ID
org.SecurityFramework.Communication.LargeImage = app.LargeImage
} else {
log.Printf("[WARNING] No handler for type %s in app framework during update of app %s", value.Type, app.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Counting up for the getting started piece
cnt := 0
if len(org.SecurityFramework.SIEM.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.Intel.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.Communication.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.Assets.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.IAM.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.Cases.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.EDR.Name) > 0 {
cnt += 1
}
if len(org.SecurityFramework.Network.Name) > 0 {
cnt += 1
}
// Add app as active for org too
if len(app.ID) > 0 && !ArrayContains(org.ActiveApps, app.ID) {
org.ActiveApps = append(org.ActiveApps, app.ID)
}
for tutorialIndex, tutorial := range org.Tutorials {
if tutorial.Name == "Find relevant apps" {
org.Tutorials[tutorialIndex].Description = fmt.Sprintf("%d out of %d apps configured. Find more relevant apps in the search bar.", cnt, 8)
if cnt > 0 {
org.Tutorials[tutorialIndex].Done = true
}
}
}
// Reset priorities as framework has changed
newPrios := []Priority{}
for _, priority := range org.Priorities {
if priority.Type == "usecase" {
continue
}
newPrios = append(newPrios, priority)
}
org.Priorities = newPrios
foundPrios, err := GetPriorities(ctx, user, org)
if err != nil {
log.Printf("[WARNING] Failed getting priorities for org %s: %s", org.Name, err)
} else {
org.Priorities = foundPrios
}
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed setting app framework for org %s: %s", org.Name, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed updating organization info. Please contact us if this persists."}`))
return
} else {
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-100"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-500"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-1000"))
DeleteCache(ctx, "all_apps")
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Username))
DeleteCache(ctx, fmt.Sprintf("user_%s", user.Id))
}
if value.ID != "remove" {
log.Printf("[DEBUG] Successfully updated app framework type %s to app %s (%s) for org %s (%s)!", value.Type, app.Name, app.ID, org.Name, org.Id)
} else {
log.Printf("[DEBUG] Successfully REMOVED app framework type %s for org %s (%s)!", value.Type, org.Name, org.Id)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
type flushWriter struct {
f http.Flusher
w io.Writer
}
func (fw *flushWriter) Write(p []byte) (n int, err error) {
n, err = fw.w.Write(p)
if fw.f != nil {
fw.f.Flush()
}
return
}
func UpdateUsecases(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get usecases. Continuing anyway: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Needs to be an active shuffler.io account to update
if project.Environment == "cloud" && !strings.HasSuffix(user.Username, "@shuffler.io") {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Can't change framework info"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Error with body read for usecase update: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
var usecase Usecase
err = json.Unmarshal(body, &usecase)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling usecase: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
usecase.Success = true
usecase.Name = strings.Replace(usecase.Name, " ", "_", -1)
usecase.Name = url.QueryEscape(usecase.Name)
log.Printf("[DEBUG] Updated usecase %s as user %s (%s)", usecase.Name, user.Username, user.Id)
usecase.EditedBy = user.Id
ctx := GetContext(request)
err = SetUsecase(ctx, usecase)
if err != nil {
log.Printf("[ERROR] Failed updating usecase: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func HandleGetUsecase(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get usecase (1). Continuing anyway: %s", err)
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false}`))
//return
}
var name string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 5 {
log.Printf("[ERROR] Path too short: %d", len(location))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
name = location[5]
}
ctx := GetContext(request)
usecase, err := GetUsecase(ctx, name)
if err != nil {
log.Printf("[ERROR] Failed getting usecase %s: %s", name, err)
usecase.Success = false
usecase.Name = name
//resp.WriteHeader(400)
//resp.Write([]byte(`{"success": false}`))
//return
} else {
usecase.Success = true
if len(usecase.Name) == 0 {
usecase.Name = name
}
}
if len(user.ActiveOrg.Id) > 0 && usecase.Name != "Reporting" && len(usecase.Name) > 3 {
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err == nil && len(org.Id) > 0 {
found := false
for _, interest := range org.Interests {
if interest.Name != usecase.Name {
continue
}
found = true
break
}
if !found {
log.Printf("[DEBUG] Updating org %s with usecase %s as interesting", user.ActiveOrg.Id, usecase.Name)
org.Interests = append(org.Interests, Priority{
Name: usecase.Name,
Description: fmt.Sprintf("User %s (%s) has shown interest in this usecase", user.Username, user.Id),
Type: "usecase",
Active: true,
Time: time.Now().Unix(),
})
SetOrg(ctx, *org, org.Id)
}
}
}
// Hardcoding until we have something good for open source + cloud
replacedName := strings.Replace(strings.ToLower(usecase.Name), " ", "_", -1)
if replacedName == "email_management" {
usecase.ExtraButtons = []ExtraButton{
ExtraButton{
Name: "IMAP",
App: "Email",
Image: "https://storage.googleapis.com/shuffle_public/app_images/email_ec25da1fdbf18934ca468788b73bec32.png",
Link: "https://shuffler.io/workflows/b65d180c-4d27-4cb6-8128-3687a08aadb3",
Type: "communication",
},
ExtraButton{
Name: "Gmail",
App: "Gmail",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Gmail_794e51c3c1a8b24b89ccc573a3defc47.png",
Link: "https://shuffler.io/workflows/e506060f-0c58-4f95-a0b8-f671103d78e5",
Type: "communication",
},
ExtraButton{
Name: "Outlook",
App: "Outlook Graph",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Outlook_graph_d71641a57deeee8149df99080adebeb7.png",
Link: "https://shuffler.io/workflows/3862ed8f-7801-4393-8524-05de8f8a401d",
Type: "communication",
},
}
} else if replacedName == "edr_to_ticket" {
usecase.ExtraButtons = []ExtraButton{
ExtraButton{
Name: "Velociraptor",
App: "Velociraptor",
Image: "https://storage.googleapis.com/shuffle_public/app_images/velociraptor_63de9fc91bcb4813d9c58cc6efd49b33.png",
Link: "https://shuffler.io/apps/63de9fc91bcb4813d9c58cc6efd49b33",
Type: "edr",
},
ExtraButton{
Name: "Carbon Black",
App: "Carbon Black",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Carbon_Black_Response_e9fa2602ea6baafffa4b5eec722095d3.png",
Link: "https://shuffler.io/apps/e9fa2602ea6baafffa4b5eec722095d3",
Type: "edr",
},
ExtraButton{
Name: "Crowdstrike",
App: "Crowdstrike",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Crowdstrike_Falcon_7a66ce3c26e0d724f31f1ebc9a7a41b4.png",
Link: "https://shuffler.io/apps/7a66ce3c26e0d724f31f1ebc9a7a41b4",
Type: "edr",
},
}
} else if replacedName == "siem_to_ticket" {
usecase.ExtraButtons = []ExtraButton{
ExtraButton{
Name: "Wazuh",
App: "Wazuh",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Wazuh_fb715a176a192687e95e9d162186c97f.png",
Link: "https://shuffler.io/workflows/bb45124c-d39e-4acc-a5d9-f8aa526042b5",
Type: "siem",
},
ExtraButton{
Name: "Splunk",
App: "Splunk",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Splunk_Splunk_e352462c6d2f0a692281600d96002a45.png",
Link: "https://shuffler.io/apps/441a2d85f6c1e8408dd1ee1e804cd241",
Type: "siem",
},
ExtraButton{
Name: "QRadar",
App: "QRadar",
Image: "https://storage.googleapis.com/shuffle_public/app_images/QRadar_4fe358bd204f672d37c55b4f1d48ccdb.png",
Link: "https://shuffler.io/apps/96a3d95a2a73cfdb51ea4a394287ed33",
Type: "siem",
},
}
} else if replacedName == "chatops" {
usecase.ExtraButtons = []ExtraButton{
ExtraButton{
Name: "Webex",
App: "Webex",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Webex_1f6f2fc4fd399597e98ff34f78f56c45.png",
Link: "https://shuffler.io/workflows/88e16093-37b7-41cf-b02b-d1ca0e737993",
Type: "communication",
},
ExtraButton{
Name: "Teams",
App: "Microsoft Teams",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Microsoft_Teams_User_Access_4826c529f8082205a4b926ac9f1dfcfb.png",
Link: "https://shuffler.io/apps/4826c529f8082205a4b926ac9f1dfcfb",
Type: "communication",
},
ExtraButton{
Name: "Slack",
App: "Slack",
Image: "https://storage.googleapis.com/shuffle_public/app_images/Slack_Web_API_f63a65ddf0ee369845b6918575d47fc1.png",
Link: "https://shuffler.io/workflows/0a7eeca9-e056-40e5-9a70-f078937c6055",
Type: "communication",
},
}
}
newjson, err := json.Marshal(usecase)
if err != nil {
log.Printf("[ERROR] Failed marshal in get usecase: %s", err)
//resp.WriteHeader(400)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking data"}`)))
//return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
// New Usecases functions
func HandlePublishUsecase(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Partner request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Unauthorized access"}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin to publish usecase: %s (%s)", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
location := strings.Split(request.URL.String(), "/")
var Id string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
Id = location[4]
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body: %v", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
var tmpData UsecaseInfo
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling usecase: %v", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Add validation for required fields
if len(tmpData.MainContent.Title) == 0 {
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "Usecase name is required"}`))
return
}
if len(tmpData.CompanyInfo.Id) == 0 {
log.Printf("[WARNING] No partner ID provided for usecase %s", tmpData.CompanyInfo.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No company ID provided"}`))
return
}
ctx := GetContext(request)
partner, err := GetPartnerById(ctx, user.ActiveOrg.Id)
if err != nil || partner == nil {
log.Printf("[WARNING] Partner doesn't exist: %v", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed finding partner"}`))
return
}
if tmpData.CompanyInfo.Id != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s (%s) is trying to publish usecase for partner %s (%s) but doesn't have access to it", user.Username, user.Id, tmpData.CompanyInfo.Name, tmpData.CompanyInfo.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Unauthorized access to partner's usecases"}`))
return
}
overwrite := false
if len(tmpData.Id) > 0 {
overwrite = true
if tmpData.Id != Id {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Usecase ID mismatch"}`))
return
}
// Validate if the org is correct in here
usecase, err := GetIndividualUsecase(ctx, tmpData.Id)
if err != nil {
log.Printf("[WARNING] Failed to get usecase by ID %s: %v", tmpData.Id, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed to get usecase by ID"}`))
return
}
if usecase.CompanyInfo.Id != user.ActiveOrg.Id {
log.Printf("[WARNING] User %s (%s) is trying to overwrite usecase for partner %s (%s) but doesn't have access to it", user.Username, user.Id, tmpData.CompanyInfo.Name, tmpData.CompanyInfo.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Unauthorized access to partner's usecases"}`))
return
}
} else {
tmpData.Id = uuid.NewV4().String()
}
tmpData.CompanyInfo.Id = user.ActiveOrg.Id
if tmpData.Public {
_, err = HandleAlgoliaUsecaseUpload(ctx, tmpData, overwrite)
if err != nil {
log.Printf("[ERROR] Failed publishing usecase to Algolia: %v", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed publishing usecase to Algolia"}`))
return
}
} else {
err = HandleAlgoliaUsecaseDeletion(ctx, tmpData.Id)
if err != nil {
log.Printf("[WARNING] Failed deleting usecase from Algolia: %v", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed deleting usecase from Algolia"}`))
return
}
}
err = SetUsecaseNew(ctx, &tmpData)
if err != nil {
log.Printf("[ERROR] Failed publishing usecase: %v", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "message": "Usecase published", "usecaseId": "` + tmpData.Id + `"}`))
}
// Used to get partner usecases (On Admin page and On Partner Page)
func HandleGetPartnerUsecases(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Partner request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in getting usecases: %s. Continuing because it may be visible to it's owner", userErr)
}
var Id string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 3 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
Id = location[4]
}
ctx := GetContext(request)
partner, err := GetPartnerById(ctx, Id)
if err != nil {
log.Printf("[ERROR] Failed to get partner: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to get partner"}`))
return
}
// Gettting the partner's usecases
var usecases []UsecaseInfo
allUsecases, err := GetPartnerUsecases(ctx, Id)
if err != nil {
log.Printf("[ERROR] Failed to get usecases: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to get usecases"}`))
return
}
// Filter to only include public usecases
if partner.Id != user.ActiveOrg.Id {
for _, usecase := range allUsecases {
if usecase.Public {
usecases = append(usecases, usecase)
}
}
} else {
usecases = allUsecases
}
type returnStruct struct {
Success bool `json:"success"`
Usecases []UsecaseInfo `json:"usecases"`
}
usecaseData := returnStruct{
Success: true,
Usecases: usecases,
}
response, err := json.Marshal(usecaseData)
if err != nil {
log.Printf("[ERROR] Failed to marshal usecases: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to process usecase data"}`))
return
}
if len(usecases) == 0 {
log.Printf("[DEBUG] No usecases found for partner %s", Id)
resp.WriteHeader(http.StatusOK)
resp.Write([]byte(`{"success": true, "usecases": []}`))
return
}
log.Printf("[DEBUG] Successfully retrieved %d usecases for %s", len(usecases), Id)
resp.WriteHeader(http.StatusOK)
resp.Write(response)
}
// Used to get the individual usecase
func HandleGetIndividualUsecase(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Partner request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in getting usecase: %s. Continuing because it may be visible to it's owner", userErr)
}
var Id string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 3 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
Id = location[4]
}
ctx := GetContext(request)
usecase, err := GetIndividualUsecase(ctx, Id)
if err != nil {
log.Printf("[ERROR] Failed to get usecase: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to get usecase"}`))
return
}
if !usecase.Public {
if usecase.CompanyInfo.Id != user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s (%s) tried to access non-public usecase %s (%s)", user.Username, user.Id, usecase.MainContent.Title, usecase.Id)
resp.WriteHeader(http.StatusForbidden)
resp.Write([]byte(`{"success": false, "reason": "This usecase is not public"}`))
return
}
}
type returnStruct struct {
Success bool `json:"success"`
Usecase UsecaseInfo `json:"usecase"`
}
usecaseData := returnStruct{
Success: true,
Usecase: usecase,
}
response, err := json.Marshal(usecaseData)
if err != nil {
log.Printf("[ERROR] Failed to marshal usecase: %v", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to process usecase data"}`))
return
}
log.Printf("[DEBUG] Successfully retrieved %s usecase of partner: %s", usecase.MainContent.Title, usecase.CompanyInfo.Id)
resp.WriteHeader(http.StatusOK)
resp.Write(response)
}
func HandleDeleteUsecase(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting Get Partner request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in delete usecase: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin to delete usecase: %s (%s)", user.Username, user.Id)
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
location := strings.Split(request.URL.String(), "/")
var Id string
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("[ERROR] Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
Id = location[4]
}
ctx := GetContext(request)
usecase, err := GetIndividualUsecase(ctx, Id)
if err != nil {
log.Printf("[ERROR] Failed getting usecase %s: %s", Id, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed getting usecase"}`))
return
}
if len(usecase.Id) == 0 {
log.Printf("[ERROR] Usecase %s not found", Id)
resp.WriteHeader(404)
resp.Write([]byte(`{"success": false, "reason": "Usecase not found"}`))
return
}
if len(usecase.CompanyInfo.Id) == 0 {
log.Printf("[WARNING] No partner ID provided for usecase %s", usecase.CompanyInfo.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No company ID provided"}`))
return
}
if usecase.CompanyInfo.Id != user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s (%s) tried to delete usecase %s from partner %s (%s) but doesn't have access", user.Username, user.Id, usecase.Id, usecase.CompanyInfo.Name, usecase.CompanyInfo.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Unauthorized access to partner's usecases"}`))
return
}
// Remove usecase from Algolia
err = HandleAlgoliaUsecaseDeletion(ctx, usecase.Id)
if err != nil {
log.Printf("[WARNING] Failed deleting usecase from Algolia: %v", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed deleting usecase from Algolia"}`))
return
}
// Delete usecase from database
nameKey := "Usecases"
DeleteCache(ctx, fmt.Sprintf("%s_partner_%s", nameKey, usecase.CompanyInfo.Id))
err = DeleteKey(ctx, nameKey, usecase.Id)
if err != nil {
log.Printf("[WARNING] Failed deleting usecase: %v", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed deleting usecase"}`))
return
}
log.Printf("[DEBUG] Successfully deleted usecase %s", usecase.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "message": "Usecase deleted successfully"}`))
return
}
func GetBackendexecution(ctx context.Context, executionId, authorization string) (WorkflowExecution, error) {
exec := WorkflowExecution{}
// Is polling the backend actually correct?
// Or should worker/backend talk to itself?
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
// Should this be without worker? :thinking:
resultUrl := fmt.Sprintf("%s/api/v1/streams/results", backendUrl)
topClient := GetExternalClient(backendUrl)
requestData := ActionResult{
ExecutionId: executionId,
Authorization: authorization,
}
data, err := json.Marshal(requestData)
if err != nil {
log.Printf("[WARNING] Failed parent init marshal: %s", err)
return exec, err
}
req, err := http.NewRequest(
"POST",
resultUrl,
bytes.NewBuffer([]byte(data)),
)
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Failed making subflow request (1): %s. Is URL valid: %s", err, resultUrl)
return exec, err
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading parent body: %s", err)
return exec, err
}
//log.Printf("BODY (%d): %s", newresp.StatusCode, string(body))
if newresp.StatusCode != 200 {
log.Printf("[ERROR] Bad statuscode getting execution (2) with URL %s: %d, %s", resultUrl, newresp.StatusCode, string(body))
return exec, errors.New(fmt.Sprintf("Bad statuscode: %d", newresp.StatusCode))
}
err = json.Unmarshal(body, &exec)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling execution: %s", err)
return exec, err
}
if exec.Status == "FINISHED" || exec.Status == "FAILURE" {
cacheKey := fmt.Sprintf("workflowexecution_%s", executionId)
err = SetCache(ctx, cacheKey, body, 31)
if err != nil {
log.Printf("[WARNING] Failed setting cache for workflowexec key %s: %s", cacheKey, err)
}
}
return exec, nil
}
func AddPriority(org Org, priority Priority, updated bool) (*Org, bool) {
found := false
usecasesFound := 0
for _, p := range org.Priorities {
if p.Type == "usecase" && p.Active {
usecasesFound += 1
}
}
for _, p := range org.Priorities {
if p.Name == priority.Name || (p.Type == priority.Type && p.Active) && p.Type != "usecase" {
found = true
break
}
if p.Type == priority.Type && p.Type == "usecase" && p.Active && usecasesFound >= 3 {
found = true
break
}
}
if !found {
priority.Active = true
org.Priorities = append(org.Priorities, priority)
updated = true
}
return &org, updated
}
// Watch academy - Shuffle 101 if you're new
// Check notifications
// Check if all apps have been discovered
// Check if notification workflow is made
// Check workflows vs usecases
// Check based on org configuration (name, environments, apps...)
// Workflows not finished / without a parent workflow/trigger
// Workflows not in production
// Should just be based on cache, not queries - keep it fast!
func GetPriorities(ctx context.Context, user User, org *Org) ([]Priority, error) {
// Get usecases -> Check which aren't done based on priorities
// 1. Check what apps are selected. Are email, edr and siem in there?
// If not - autocorrect based on workflows' apps?
//log.Printf("[DEBUG] SecurityFramework: %s", org.SecurityFramework)
// First prio: Find these & attach usecases?
// Only set these if cache is set for the user
orgUpdated := false
updated := false
if project.CacheDb == false {
// Not checking as cache is used for all checks
return org.Priorities, nil
}
if len(org.Defaults.NotificationWorkflow) == 0 {
org, updated = AddPriority(*org, Priority{
Name: fmt.Sprintf("You haven't defined a notification workflow yet."),
Description: "Notification workflows are used to automate your notification handling. These can be used to alert yourself in other systems when issues are found in your current- or sub-organizations",
Type: "notifications",
Active: true,
URL: fmt.Sprintf("/admin?admin_tab=organization"),
Severity: 2,
}, updated)
if updated {
orgUpdated = true
}
}
// Notify about hybrid
if project.Environment == "cloud" {
org, updated = AddPriority(*org, Priority{
Name: fmt.Sprintf("Try Hybrid Shuffle by connecting environments"),
Description: "Hybrid Shuffle allows you to connect Shuffle to your local datacenter(s) internal resources, and get the results in the cloud.",
Type: "hybrid",
Active: true,
URL: fmt.Sprintf("/admin?tab=environments"),
Severity: 3,
}, updated)
} else {
org, updated = AddPriority(*org, Priority{
Name: fmt.Sprintf("Get more functionality by connecting to the cloud"),
Description: "Get access to webhooks, schedules and other functions by connecting to the cloud. Try it out for free, or contact our team if you want to learn more!",
Type: "hybrid",
Active: true,
URL: fmt.Sprintf("/admin?admin_tab=cloud_sync"),
Severity: 2,
}, updated)
}
var notifications []Notification
cache, err := GetCache(ctx, fmt.Sprintf("notifications_%s", org.Id))
if err == nil {
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, ¬ifications)
if err == nil && len(notifications) > 0 {
org, updated = AddPriority(*org, Priority{
Name: fmt.Sprintf("You have %d unhandled notifications.", len(notifications)),
Description: "Notifications help make your workflow infrastructure stable. Click the notification icon in the top right to see all open ones.",
Type: "notifications",
Active: true,
URL: fmt.Sprintf("/admin?tab=priorities"),
Severity: 1,
}, updated)
if updated {
orgUpdated = true
}
}
} else {
}
if len(org.MainPriority) == 0 {
// Just choosing something for them, e.g. basic usecase building
org.MainPriority = "1. Collect"
orgUpdated = true
}
org, orgUpdated = GetWorkflowSuggestions(ctx, user, org, orgUpdated, 1)
if orgUpdated {
log.Printf("[DEBUG] Should update org with %d priorities", len(org.Priorities))
SetOrg(ctx, *org, org.Id)
}
return org.Priorities, nil
}
// Sorts an org list in order to make ChildOrgs appear under their parent org
func SortOrgList(orgs []OrgMini) []OrgMini {
// Sort based on the name of the org first
sort.Slice(orgs, func(i, j int) bool {
return strings.ToLower(orgs[i].Name) < strings.ToLower(orgs[j].Name)
})
// Creates parentorg map
parentOrgs := map[string][]OrgMini{}
for _, org := range orgs {
if len(org.CreatorOrg) == 0 && len(org.ChildOrgs) > 0 {
parentOrgs[org.Id] = []OrgMini{}
} else if len(org.CreatorOrg) == 0 {
// No childorgs, but isn't a parentorg either
parentOrgs[org.Id] = []OrgMini{}
} else {
// Child orgs go here
}
}
noParentOrg := []OrgMini{}
for _, org := range orgs {
// Check if parent in parentOrgs map
if len(org.CreatorOrg) == 0 {
continue
}
if val, ok := parentOrgs[org.CreatorOrg]; ok {
parentOrgs[org.CreatorOrg] = append(val, org)
} else {
noParentOrg = append(noParentOrg, org)
}
}
newOrgs := []OrgMini{}
for key, value := range parentOrgs {
// Find key in orgs
found := false
for _, org := range orgs {
if org.Id == key {
found = true
newOrgs = append(newOrgs, org)
break
}
}
if found {
for _, childorg := range value {
newOrgs = append(newOrgs, childorg)
}
}
}
// Adding orgs where parentorg is unavailable
// They should probably be under some "inactive" parentorg..
newOrgs = append(newOrgs, noParentOrg...)
return newOrgs
}
func findMissingChildren(ctx context.Context, workflowExecution *WorkflowExecution, children map[string][]string, inputNode string, checkedNodes []string) []string {
nextActions := []string{}
if ArrayContains(checkedNodes, inputNode) {
return nextActions
}
checkedNodes = append(checkedNodes, inputNode)
parentRan := false
for _, result := range workflowExecution.Results {
if result.Action.ID == inputNode && result.Status != "WAITING" {
parentRan = true
}
}
if !parentRan {
//log.Printf("[ERROR] Parent node %s hasn't run yet, skipping children search", inputNode)
return []string{inputNode}
} else {
// Starting from the startnode, go through the workflow one level at a time
foundCnt := 0
for _, child := range children[inputNode] {
// Check if the parent and its childs have a result
found := false
for _, result := range workflowExecution.Results {
if result.Action.ID == child {
foundCnt += 1
found = true
break
}
}
if !found {
// Due to being too fast cleared
cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, child)
_, err := GetCache(ctx, cacheId)
if err != nil {
nextActions = append(nextActions, child)
}
}
}
if foundCnt == len(children[inputNode]) {
//log.Printf("[DEBUG] All nodes done (%s). Check their child results. Child nodes: %d, found: %d", inputNode, len(children[inputNode]), foundCnt)
// Randomize order as to keep digging
nextActions = []string{}
for _, child := range children[inputNode] {
next := findMissingChildren(ctx, workflowExecution, children, child, checkedNodes)
if len(next) > 0 {
// WARNING: Do NOT break here.
// This has caused bugs before in complicated decision trees.
nextActions = append(nextActions, next...)
}
}
} else {
if debug {
//log.Printf("[DEBUG] Missing nodes (%s). Found: %d, Expected: %d. NEXT: %#v", inputNode, foundCnt, len(children[inputNode]), nextActions)
}
}
}
// Dedup here
newList := []string{}
for _, next := range nextActions {
if ArrayContains(newList, next) {
continue
}
newList = append(newList, next)
}
return newList
}
// Finds next actions that aren't already executed and don't have results
func CheckNextActions(ctx context.Context, workflowExecution *WorkflowExecution) []string {
extra := 0
parents := map[string][]string{}
children := map[string][]string{}
nextActions := []string{}
inputNode := workflowExecution.Start
if len(workflowExecution.Results) == 0 {
return []string{inputNode}
}
if ValidateFinished(ctx, extra, *workflowExecution) {
return []string{}
}
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.TriggerType != "SUBFLOW" && trigger.TriggerType != "USERINPUT" {
continue
}
extra += 1
}
for _, branch := range workflowExecution.Workflow.Branches {
// Check what the parent is first. If it's trigger - skip
sourceFound := false
destinationFound := false
for _, action := range workflowExecution.Workflow.Actions {
if action.ID == branch.SourceID {
sourceFound = true
}
if action.ID == branch.DestinationID {
destinationFound = true
}
}
if !sourceFound || !destinationFound {
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.ID == branch.SourceID {
sourceFound = true
}
if trigger.ID == branch.DestinationID {
destinationFound = true
}
}
}
foundCnt := 0
if sourceFound {
parents[branch.DestinationID] = append(parents[branch.DestinationID], branch.SourceID)
foundCnt += 1
}
if destinationFound {
children[branch.SourceID] = append(children[branch.SourceID], branch.DestinationID)
foundCnt += 1
}
if foundCnt != 2 {
//if debug {
// log.Printf("[ERROR] Missing branch fullfillment for src + dst! Source: %s, Destination: %s, Branch: %#v", branch.SourceID, branch.DestinationID, branch)
//}
}
}
nextActions = findMissingChildren(ctx, workflowExecution, children, inputNode, []string{})
// SHOULD WE: Write code here which returns IF an action should be SKIPPED. If ALL parents are SKIPPED/FAILED, return something like []string{id:SKIPPED} -> parent function that calls this should make it SKIPPED
// Question: Should we just run SKIPPED requests directly from here, then NOT return the ID? Maybe set it in cache?
var updatedActions []string
for _, actionId := range nextActions {
skippedParents := 0
if _, ok := parents[actionId]; !ok {
updatedActions = append(updatedActions, actionId)
continue
}
for _, parent := range parents[actionId] {
_, result := GetActionResult(ctx, *workflowExecution, parent)
if result.Status == "SKIPPED" {
skippedParents += 1
}
}
if skippedParents >= len(parents[actionId]) && actionId != workflowExecution.Start {
for _, action := range workflowExecution.Workflow.Actions {
if actionId != action.ID {
continue
}
foundAction := GetAction(*workflowExecution, actionId, action.Environment)
err := ActionSkip(ctx, foundAction, workflowExecution, parents[actionId])
if err != nil {
log.Printf("[ERROR][%s] Failed to skip action %s (%s): %s", workflowExecution.ExecutionId, action.Label, action.ID, err)
continue
}
}
} else {
updatedActions = append(updatedActions, actionId)
}
}
return updatedActions
}
func ActionSkip(ctx context.Context, foundAction Action, exec *WorkflowExecution, parent []string) error {
_, actionResult := GetActionResult(ctx, *exec, foundAction.ID)
if actionResult.Action.ID == foundAction.ID {
log.Printf("[DEBUG][%s] Result already exist for the action %s (%s)", exec.ExecutionId, foundAction.Label, foundAction.ID)
return nil
}
newResult := ActionResult{
Action: foundAction,
ExecutionId: exec.ExecutionId,
Authorization: exec.Authorization,
Result: fmt.Sprintf(`{"success": false, "reason": "Skipped because of previous node - %d - %v"}`, len(parent), parent),
StartedAt: 0,
CompletedAt: 0,
Status: "SKIPPED",
}
resultData, err := json.Marshal(newResult)
if err != nil {
return err
}
streamUrl := fmt.Sprintf("http://localhost:5001/api/v1/streams")
if project.Environment == "cloud" {
streamUrl = fmt.Sprintf("https://shuffler.io/api/v1/streams")
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
streamUrl = fmt.Sprintf("https://%s.%s.r.appspot.com/api/v1/streams", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
streamUrl = fmt.Sprintf("%s/api/v1/streams", os.Getenv("SHUFFLE_CLOUDRUN_URL"))
}
} else {
if len(os.Getenv("WORKER_HOSTNAME")) > 0 {
streamUrl = fmt.Sprintf("http://%s:33333/api/v1/streams", os.Getenv("WORKER_HOSTNAME"))
}
if os.Getenv("SHUFFLE_OPTIMIZED") == "true" && len(os.Getenv("WORKER_PORT")) > 0 {
streamUrl = fmt.Sprintf("http://localhost:%s/api/v1/streams", os.Getenv("WORKER_PORT"))
} else if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && (project.Environment == "" || project.Environment == "worker") {
streamUrl = fmt.Sprintf("http://localhost:33333/api/v1/streams")
} else {
if len(os.Getenv("BASE_URL")) > 0 {
streamUrl = fmt.Sprintf("%s/api/v1/streams", os.Getenv("BASE_URL"))
}
}
}
//log.Printf("[DEBUG] Sending skip for action %s (%s) to URL %s", foundAction.Label, foundAction.AppName, streamUrl)
req, err := http.NewRequest(
"POST",
streamUrl,
bytes.NewBuffer([]byte(resultData)),
)
if err != nil {
log.Printf("[ERROR] Error building SKIPPED request (%s): %s", foundAction.Label, err)
return err
}
client := &http.Client{}
newresp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Error running SKIPPED request (%s): %s", foundAction.Label, err)
return err
}
defer newresp.Body.Close()
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading body when running SKIPPED request (%s): %s", foundAction.Label, err)
return err
}
//log.Printf("[DEBUG] Skipped body return from %s (%d): %s", streamUrl, newresp.StatusCode, string(body))
if strings.Contains(string(body), "already finished") {
log.Printf("[WARNING] Data couldn't be re-inputted for %s.", foundAction.Label)
// DONT CHANGE THE ERROR OUTPUT HERE
}
return nil
}
// Decideds what should happen next. Used both for cloud & onprem environments
// Added early 2023 as yet another way to standardize decisionmaking of app executions
func DecideExecution(ctx context.Context, workflowExecution WorkflowExecution, environment string) (WorkflowExecution, []Action) {
// ensuring always latest
newexec, err := GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
if err != nil {
log.Printf("[ERROR] Failed to get workflow execution in Decide: %s", err)
} else {
workflowExecution = *newexec
}
startAction, extra, children, parents, visited, executed, nextActions, environments := GetExecutionVariables(ctx, workflowExecution.ExecutionId)
if len(startAction) == 0 {
startAction = workflowExecution.Start
if len(startAction) == 0 {
log.Printf("[WARNING] Didn't find execution start action. Setting it to workflow start action.")
startAction = workflowExecution.Workflow.Start
}
}
if len(nextActions) == 0 {
nextActions = CheckNextActions(ctx, &workflowExecution)
}
// Dedup results just in case
newResults := []ActionResult{}
handled := []string{}
for _, result := range workflowExecution.Results {
if ArrayContains(handled, result.Action.ID) {
continue
}
handled = append(handled, result.Action.ID)
newResults = append(newResults, result)
}
workflowExecution.Results = newResults
relevantActions := []Action{}
// Validates RERUN of single actions (new 2025)
// Identified by:
// 1. Predefined result from previous exec
// 2. Only ONE action
// 3. Every predefined result having result.Action.Category == "rerun"
if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 {
finished := ValidateFinished(ctx, extra, workflowExecution)
if finished {
return workflowExecution, relevantActions
}
}
log.Printf("[INFO][%s] Inside Decide execution with %d / %d results (extra: %d). Status: %s", workflowExecution.ExecutionId, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra, extra, workflowExecution.Status)
if len(startAction) == 0 {
startAction = workflowExecution.Start
if len(startAction) == 0 {
log.Printf("[WARNING] Didn't find execution start action. Setting it to workflow start action (%s)", workflowExecution.Workflow.Start)
startAction = workflowExecution.Workflow.Start
workflowExecution.Start = workflowExecution.Workflow.Start
}
}
queueNodes := []string{}
if len(workflowExecution.Results) == 0 {
nextActions = []string{startAction}
} else {
// This is to re-check the nodes that exist and whether they should continue
appendActions := []string{}
for _, item := range workflowExecution.Results {
// FIXME: Check whether the item should be visited or not
// Do the same check as in walkoff.go - are the parents done?
// If skipped and both parents are skipped: keep as skipped, otherwise queue
if item.Status == "SKIPPED" {
isSkipped := true
for _, branch := range workflowExecution.Workflow.Branches {
// 1. Finds branches where the destination is our node
// 2. Finds results of those branches, and sees the status
// 3. If the status isn't skipped or failure, then it will still run this node
if branch.DestinationID == item.Action.ID {
for _, subresult := range workflowExecution.Results {
if subresult.Action.ID == branch.SourceID {
if subresult.Status != "SKIPPED" && subresult.Status != "FAILURE" {
isSkipped = false
break
}
}
}
}
}
if isSkipped {
//log.Printf("Skipping %s as all parents are done", item.Action.Label)
if !ArrayContains(visited, item.Action.ID) {
//log.Printf("[INFO] Adding visited (1): %s", item.Action.Label)
visited = append(visited, item.Action.ID)
}
} else {
//log.Printf("[INFO] Continuing %s as all parents are NOT done", item.Action.Label)
// FIXME: Remove this visited?
//visited = append(visited, item.Action.ID)
appendActions = append(appendActions, item.Action.ID)
}
} else {
if item.Status == "FINISHED" {
//log.Printf("[INFO] Adding visited (2): %s", item.Action.Label)
visited = append(visited, item.Action.ID)
}
}
//if len(nextActions) == 0 {
//nextActions = append(nextActions, children[item.Action.ID]...)
for _, child := range children[item.Action.ID] {
if !ArrayContains(nextActions, child) && !ArrayContains(visited, child) && !ArrayContains(visited, child) {
nextActions = append(nextActions, child)
}
}
if len(appendActions) > 0 {
//log.Printf("APPENDED NODES: %s", appendActions)
nextActions = append(nextActions, appendActions...)
}
}
}
//log.Printf("Nextactions: %s", nextActions)
// This is a backup in case something goes wrong in this complex hellhole.
// Max default execution time is 5 minutes for now anyway, which should take
// care if it gets stuck in a loop.
// FIXME: Force killing a worker should result in a notification somewhere
if len(nextActions) == 0 {
if project.Environment != "cloud" || len(workflowExecution.Results) != len(workflowExecution.Workflow.Actions) {
log.Printf("[DEBUG][%s] No next action. Finished? Result vs Actions: %d - %d", workflowExecution.ExecutionId, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
}
extra = 0
for _, trigger := range workflowExecution.Workflow.Triggers {
if (trigger.Name == "User Input" && trigger.AppName == "User Input") || (trigger.Name == "Shuffle Workflow" && trigger.AppName == "Shuffle Workflow") {
extra += 1
}
}
exit := true
for _, item := range workflowExecution.Results {
if item.Status == "EXECUTING" {
exit = false
break
}
}
if len(environments) == 1 {
log.Printf("[INFO] Should send results to the backend because environments are %s", environments)
ValidateFinished(ctx, extra, workflowExecution)
}
if exit && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) {
ValidateFinished(ctx, extra, workflowExecution)
//handleAbortExecution(ctx, workflowExecution)
return workflowExecution, relevantActions
}
// Look for the NEXT missing action
notFound := []string{}
for _, action := range workflowExecution.Workflow.Actions {
found := false
for _, result := range workflowExecution.Results {
if action.ID == result.Action.ID {
found = true
break
}
}
if !found {
notFound = append(notFound, action.ID)
}
}
//log.Printf("SOMETHING IS MISSING!: %s", notFound)
for _, item := range notFound {
if ArrayContains(executed, item) {
log.Printf("%s has already executed but no result!", item)
}
// Visited means it's been touched in any way.
outerIndex := -1
for index, visit := range visited {
if visit == item {
outerIndex = index
break
}
}
if outerIndex >= 0 {
log.Printf("Removing index %s from visited", item)
visited = append(visited[:outerIndex], visited[outerIndex+1:]...)
}
fixed := 0
for _, parent := range parents[item] {
parentResult := ActionResult{}
workflowExecution, parentResult = GetActionResult(ctx, workflowExecution, parent)
if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" || parentResult.Status == "SKIPPED" || parentResult.Status == "FAILURE" {
fixed += 1
}
}
if fixed == len(parents[item]) {
nextActions = append(nextActions, item)
}
// If it's not executed and not in nextActions
}
}
//log.Printf("Checking nextactions: %s", nextActions)
for _, node := range nextActions {
nodeChildren := children[node]
for _, child := range nodeChildren {
if !ArrayContains(queueNodes, child) {
queueNodes = append(queueNodes, child)
}
}
}
// IF NOT VISITED && IN toExecuteOnPrem
// SKIP if it's not onprem
for _, nextAction := range nextActions {
//log.Printf("[DEBUG] Handling nextAction %s", nextAction)
action := GetAction(workflowExecution, nextAction, environment)
// Using cache to ensure the same app isn't ran twice
// May arise due to one app being nanoseconds before another
// Not really sure how this edgecase happens.
// FIXME
// Execute, as we don't really care if env is not set? IDK
if action.Environment != environment { //&& action.Environment != "" {
if strings.ToLower(action.Environment) == strings.ToLower(environment) {
// Fixing names
action.Environment = environment
} else {
//log.Printf("envs: %s", environments)
//log.Printf("[WARNING] Bad environment for node: %s. Want %s", action.Environment, environment)
action.Environment = "cloud"
//DeleteCache(ctx, newExecId)
//continue
}
}
// check whether the parent is finished executing
fixed := 0
fixedNames := []string{}
continueOuter := true
if action.IsStartNode {
continueOuter = false
} else if len(parents[nextAction]) > 0 {
// Wait for parents to finish executing
skippedCnt := 0
childNodes := FindChildNodes(workflowExecution.Workflow, nextAction, []string{}, []string{})
for _, parent := range parents[nextAction] {
// Check if the parent is also a child. This can ensure continueation no matter what
if ArrayContains(childNodes, parent) {
log.Printf("[ERROR][%s] Parent %s is also a child of %s. Skipping parent check", workflowExecution.ExecutionId, parent, nextAction)
fixed += 1
continue
}
// Not including ABORTED/FAILURE
_, parentResult := GetActionResult(ctx, workflowExecution, parent)
if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" || parentResult.Status == "SKIPPED" {
if parentResult.Status == "SKIPPED" {
skippedCnt += 1
}
fixed += 1
// Debug names
fixedNames = append(fixedNames, fmt.Sprintf("%s:%s", parent, parentResult.Status))
} else {
// Should check if it's actually RAN at all?
// This is not necessary anymore as the cache is used previously, and this won't be any different
//Look for ABORT/FAILURE?
//parentId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, parent)
//_, err := GetCache(ctx, parentId)
//if err != nil {
// //log.Printf("[INFO] No cache for parent ID %#v", parentId)
//} else {
// //log.Printf("Parent ID already ran. How long ago?")
//}
}
}
// Check if there are as many successful results as there are parents
// Else, continueOuter = true by default, and it will be skipped
if fixed == len(parents[nextAction]) {
continueOuter = false
if fixed > 0 && skippedCnt == len(parents[nextAction]) {
//log.Printf("[WARNING][%s] All parents of %s (%s) are skipped. (%d/%d): %s. Should set to skipped.", workflowExecution.ExecutionId, action.Label, nextAction, fixed, len(parents[nextAction]), strings.Join(parents[nextAction], ", "))
continueOuter = true
}
}
} else {
//log.Printf("[INFO] No parents for %s", action.Label)
continueOuter = false
}
if continueOuter {
//log.Printf("[DEBUG][%s] Parents of %s (%s) aren't finished yet (%d/%d). Parents: %s", workflowExecution.ExecutionId, action.Label, nextAction, fixed, len(parents[nextAction]), strings.Join(parents[nextAction], ", "))
continue
} else {
// Was a bug related to bad parent
if len(parents[nextAction]) > 0 {
//log.Printf("[DEBUG][%s] ALL Parents of %s (%s) are finished. (%d/%d): %s. But are they succeeded?", workflowExecution.ExecutionId, action.Label, nextAction, fixed, len(parents[nextAction]), strings.Join(parents[nextAction], ", "))
}
}
// get action status
workflowExecution, actionResult := GetActionResult(ctx, workflowExecution, nextAction)
if actionResult.Action.ID == action.ID {
//log.Printf("\n\n[INFO] %s (%s) already has status %s\n\n", action.Label, action.ID, actionResult.Status)
//DeleteCache(ctx, newExecId)
continue
} else {
}
// Checked multiple times due to the cache
newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, nextAction)
_, err := GetCache(ctx, newExecId)
if err == nil {
//log.Printf("\n\n[DEBUG] Already found %s - returning\n\n", newExecId)
continue
}
parentlen := 0
// Check if nextAction in parents map, not len of it
if _, ok := parents[nextAction]; ok {
parentlen = len(parents[nextAction])
}
//log.Printf("[DEBUG][%s] Running %s (%s) with %d parent(s). Names: %#v", workflowExecution.ExecutionId, action.Label, nextAction, parentlen, fixedNames)
if project.Environment != "cloud" {
branchesFound := 0
parentFinished := 0
for _, item := range workflowExecution.Workflow.Branches {
if item.DestinationID != action.ID {
continue
}
branchesFound += 1
found := false
for _, result := range workflowExecution.Results {
if result.Action.ID != item.SourceID {
continue
}
found = true
// Check for fails etc
if result.Status == "SUCCESS" || result.Status == "SKIPPED" {
parentFinished += 1
} else {
log.Printf("[WARNING] Parent %s has status %s", result.Action.Label, result.Status)
}
break
}
if !found {
// Ensuring triggers are handled as they should
for _, trigger := range workflowExecution.Workflow.Triggers {
if trigger.AppName == "Shuffle Workflow" || trigger.AppName == "User Input" || trigger.AppName == "shuffle-subworkflow" {
continue
}
if trigger.ID == item.SourceID {
found = true
parentFinished += 1
}
}
}
}
if branchesFound != parentFinished {
log.Printf("[WARNING][%s] Skipping execution of %s (%s) due to unfinished parents (%d/%d). Orig parentlen: %d", workflowExecution.ExecutionId, action.Label, nextAction, parentFinished, branchesFound, parentlen)
continue
}
}
if action.AppName == "Shuffle Workflow" {
branchesFound := 0
parentFinished := 0
for _, item := range workflowExecution.Workflow.Branches {
if item.DestinationID == action.ID {
branchesFound += 1
for _, result := range workflowExecution.Results {
if result.Action.ID == item.SourceID {
// Check for fails etc
if result.Status == "SUCCESS" || result.Status == "SKIPPED" {
parentFinished += 1
} else {
log.Printf("Parent %s has status %s", result.Action.Label, result.Status)
}
break
}
}
}
}
if branchesFound == parentFinished {
action.Environment = environment
action.AppName = "shuffle-subflow"
action.Name = "run_subflow"
action.AppVersion = "1.1.0"
//appname := action.AppName
//appversion := action.AppVersion
//appname = strings.Replace(appname, ".", "-", -1)
//appversion = strings.Replace(appversion, ".", "-", -1)
//visited = append(visited, action.ID)
//executed = append(executed, action.ID)
trigger := Trigger{}
for _, innertrigger := range workflowExecution.Workflow.Triggers {
if innertrigger.ID == action.ID {
trigger = innertrigger
break
}
}
// FIXME: Add startnode from frontend
action.ExecutionDelay = trigger.ExecutionDelay
action.Parameters = []WorkflowAppActionParameter{}
for _, parameter := range trigger.Parameters {
parameter.Variant = "STATIC_VALUE"
action.Parameters = append(action.Parameters, parameter)
}
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_workflow",
Value: workflowExecution.Workflow.ID,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_execution",
Value: workflowExecution.ExecutionId,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_auth",
Value: workflowExecution.Authorization,
})
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_node",
Value: action.ID,
})
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
if len(backendUrl) > 0 {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "backend_url",
Value: backendUrl,
})
}
}
} else if action.AppName == "User Input" {
branchesFound := 0
parentFinished := 0
for _, item := range workflowExecution.Workflow.Branches {
if item.DestinationID == action.ID {
branchesFound += 1
for _, result := range workflowExecution.Results {
if result.Action.ID == item.SourceID {
// Check for fails etc
if result.Status == "SUCCESS" || result.Status == "SKIPPED" {
parentFinished += 1
} else {
log.Printf("Parent %s has status %s", result.Action.Label, result.Status)
}
break
}
}
}
}
if branchesFound == parentFinished {
if action.ID == workflowExecution.Start {
log.Printf("Skipping because it's the startnode")
visited = append(visited, action.ID)
executed = append(executed, action.ID)
continue
} else {
log.Printf("[DEBUG][%s] Should stop after this iteration because it's user-input based.", workflowExecution.ExecutionId)
trigger := Trigger{}
for _, innertrigger := range workflowExecution.Workflow.Triggers {
if innertrigger.ID == action.ID {
trigger = innertrigger
break
}
}
trigger.LargeImage = ""
triggerData, err := json.Marshal(trigger)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling action: %s", err)
triggerData = []byte("Failed unmarshalling. Cancel execution!")
}
_ = triggerData
timeNow := int64(time.Now().Unix())
result := ActionResult{
Action: action,
ExecutionId: workflowExecution.ExecutionId,
Authorization: workflowExecution.Authorization,
Result: "{\"success\": true, \"reason\": \"WAITING FOR USER INPUT\"}",
StartedAt: (timeNow + 3) * 1000,
CompletedAt: (timeNow + 3) * 1000,
Status: "WAITING",
}
// old: 1710863749000
// new: 1710863808000
workflowExecution.Results = append(workflowExecution.Results, result)
workflowExecution.Status = "WAITING"
err = SetWorkflowExecution(ctx, workflowExecution, true)
if err != nil {
log.Printf("[ERROR] Error saving workflow execution actionresult setting: %s", err)
break
}
action.Environment = environment
action.AppName = "shuffle-subflow"
action.Name = "run_userinput"
action.AppVersion = "1.1.0"
action.ExecutionDelay = trigger.ExecutionDelay
for _, innertrigger := range workflowExecution.Workflow.Triggers {
if innertrigger.ID == action.ID {
trigger = innertrigger
break
}
}
smsEnabled := false
emailEnabled := false
subflowEnabled := false
for _, trigger := range trigger.Parameters {
if trigger.Name == "type" {
if strings.Contains(trigger.Value, "sms") {
smsEnabled = true
}
if strings.Contains(trigger.Value, "email") {
emailEnabled = true
}
if strings.Contains(trigger.Value, "subflow") {
subflowEnabled = true
}
}
}
//log.Printf("\n\nSHOULD RUN USER INPUT! SMS: %#v, email: %#v, subflow: %#v \n\n", smsEnabled, emailEnabled, subflowEnabled)
// FIXME: Add startnode from frontend
action.ExecutionDelay = trigger.ExecutionDelay
action.Parameters = []WorkflowAppActionParameter{}
for _, parameter := range trigger.Parameters {
parameter.Variant = "STATIC_VALUE"
if parameter.Name == "alertinfo" {
parameter.Name = "information"
}
if parameter.Name == "sms" && smsEnabled == false {
continue
}
if parameter.Name == "email" && emailEnabled == false {
continue
}
if parameter.Name == "subflow" && subflowEnabled == false {
continue
}
action.Parameters = append(action.Parameters, parameter)
}
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "startnode",
Value: workflowExecution.Start,
})
backendUrl := os.Getenv("BASE_URL")
if len(os.Getenv("SHUFFLE_GCEPROJECT")) > 0 && len(os.Getenv("SHUFFLE_GCEPROJECT_LOCATION")) > 0 {
backendUrl = fmt.Sprintf("https://%s.%s.r.appspot.com", os.Getenv("SHUFFLE_GCEPROJECT"), os.Getenv("SHUFFLE_GCEPROJECT_LOCATION"))
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
backendUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
// Fallback
if len(backendUrl) == 0 {
backendUrl = "https://shuffler.io"
}
if len(backendUrl) > 0 {
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "backend_url",
Value: backendUrl,
})
}
log.Printf("[DEBUG][%s] Starting with user input sourcenode '%s'", workflowExecution.ExecutionId, trigger.ID)
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "source_node",
Value: trigger.ID,
})
// If sms/email, it should be setting the apikey based on the org
syncApikey := workflowExecution.Authorization
if project.Environment != "cloud" && project.Environment != "worker" {
org, err := GetOrg(ctx, workflowExecution.ExecutionOrg)
if err == nil {
log.Printf("[DEBUG] Got syncconfig key: %s", org.SyncConfig.Apikey)
syncApikey = org.SyncConfig.Apikey
} else {
log.Printf("[ERROR] Failed to get org %s: %s", workflowExecution.ExecutionOrg, err)
}
}
action.Parameters = append(action.Parameters, WorkflowAppActionParameter{
Name: "user_apikey",
Value: syncApikey,
})
}
}
} else {
//log.Printf("Handling action %s", action)
}
// Here it's still in a loop..?
_, _, _, _, _, executed, _, _ = GetExecutionVariables(ctx, workflowExecution.ExecutionId)
if ArrayContains(visited, action.ID) || ArrayContains(executed, action.ID) {
log.Printf("[WARNING][%s] SKIP EXECUTION %s:%s with label %s", workflowExecution.ExecutionId, action.AppName, action.AppVersion, action.Label)
continue
} else {
// FIXME? This was a test to check if a result was finished or not after a certain time. Not viable for production (obv)
//time.Sleep(1 * time.Second)
//validateExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
//if err == nil {
// skipAction := false
// for _, result := range validateExecution.Results {
// if result.Action.ID == action.ID {
// skipAction = true
// break
// }
// }
// if skipAction {
// log.Printf("[DEBUG] Skipping action %s afterall.", action.Label)
// continue
// }
//}
}
// Verify if parents are done
relevantActions = append(relevantActions, action)
}
return workflowExecution, relevantActions
}
func isNoProxyHost(noProxy, inputHost string) bool {
// Normalize the host by removing the port if present
host, _, err := net.SplitHostPort(inputHost)
if err != nil {
//log.Printf("[ERROR] Failed to split host and port: %s", err)
host = inputHost
}
host = strings.TrimSpace(host) // Fallback to trimming
noProxies := strings.Split(noProxy, ",")
for _, noProxyEntry := range noProxies {
newProxyEntry, _, err := net.SplitHostPort(noProxyEntry)
if err != nil {
//log.Printf("[ERROR] Failed to split host and port for NOPROXY: %s", err)
} else {
noProxyEntry = newProxyEntry
}
noProxyEntry = strings.TrimSpace(noProxyEntry)
// Handle wildcards or suffix matching
if strings.HasPrefix(noProxyEntry, ".") {
if strings.HasSuffix(host, noProxyEntry) || host == noProxyEntry[1:] {
return true
}
} else if host == noProxyEntry {
// Exact match
return true
} else if ip := net.ParseIP(noProxyEntry); ip != nil {
// Handle exact IP matches
if ip.Equal(net.ParseIP(host)) {
return true
}
}
}
return false
}
func GetExternalClient(baseUrl string) *http.Client {
// Look for internal proxy instead
// in case apps need a different one: https://jamboard.google.com/d/1KNr4JJXmTcH44r5j_5goQYinIe52lWzW-12Ii_joi-w/viewer?mtt=9r8nrqpnbz6z&f=0
//httpProxy := os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY")
//httpsProxy := os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY")
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
noProxy := os.Getenv("NO_PROXY")
if len(os.Getenv("NOPROXY")) > 0 {
noProxy = os.Getenv("NOPROXY")
os.Setenv("NO_PROXY", noProxy)
}
if len(noProxy) > 0 {
os.Setenv("no_proxy", noProxy)
}
// Check if the IP in the baseUrl is a local one
parsedUrl, err := url.Parse(baseUrl)
backendUrl := os.Getenv("BASE_URL")
parsedBackendUrl, err := url.Parse(backendUrl)
if err == nil && project.Environment != "cloud" {
// Check if host has shuffle- as prefix OR uses a shuffle-specific port
// Check until 33350 (Orborus -> Worker and Worker -> Apps)
if strings.HasPrefix(parsedUrl.Host, "shuffle-") || parsedUrl.Port() == "33333" || parsedUrl.Port() == "33334" || parsedUrl.Port() == "33335" || parsedUrl.Port() == "33336" || parsedUrl.Port() == "33337" || parsedUrl.Port() == "33338" || parsedUrl.Port() == "33339" || parsedUrl.Port() == "33340" || parsedUrl.Port() == "33341" || parsedUrl.Port() == "33342" || parsedUrl.Port() == "33343" || parsedUrl.Port() == "33344" || parsedUrl.Port() == "33345" || parsedUrl.Port() == "33346" || parsedUrl.Port() == "33347" || parsedUrl.Port() == "33348" || parsedUrl.Port() == "33349" || parsedUrl.Port() == "33350" || parsedBackendUrl.Host == parsedUrl.Host {
//log.Printf("[INFO] Running with internal proxy for %s", parsedUrl)
httpProxy = os.Getenv("SHUFFLE_INTERNAL_HTTP_PROXY")
httpsProxy = os.Getenv("SHUFFLE_INTERNAL_HTTPS_PROXY")
if len(os.Getenv("SHUFFLE_INTERNAL_NO_PROXY")) > 0 {
noProxy = os.Getenv("SHUFFLE_INTERNAL_NO_PROXY")
}
if len(os.Getenv("SHUFFLE_INTERNAL_NOPROXY")) > 0 {
noProxy = os.Getenv("SHUFFLE_INTERNAL_NOPROXY")
}
}
}
// Manage noproxy manually
if len(noProxy) > 0 {
isNoProxy := isNoProxyHost(noProxy, parsedUrl.Host)
if isNoProxy {
log.Printf("[INFO] Skipping proxy for %s", parsedUrl)
httpProxy = ""
httpsProxy = ""
}
}
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.MaxIdleConnsPerHost = 100
transport.ResponseHeaderTimeout = time.Second * 60
transport.IdleConnTimeout = time.Second * 60
transport.Proxy = nil
skipSSLVerify := false
if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" || strings.ToLower(os.Getenv("SHUFFLE_SKIPSSL_VERIFY")) == "true" {
//log.Printf("[DEBUG] SKIPPING SSL verification with Opensearch")
skipSSLVerify = true
os.Setenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY", "true")
os.Setenv("SHUFFLE_SKIPSSL_VERIFY", "true")
}
transport.TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS11,
InsecureSkipVerify: skipSSLVerify,
}
if project.Environment != "cloud" {
rootCAs, _ := x509.SystemCertPool()
if rootCAs == nil {
rootCAs = x509.NewCertPool()
}
certDir := "/certs/"
if os.Getenv("SHUFFLE_CERT_DIR") != "" {
certDir = os.Getenv("SHUFFLE_CERT_DIR")
log.Printf("[INFO] Reading self signed certificates from custom dir '%s'", certDir)
}
files, err := os.ReadDir(certDir)
if err == nil && os.Getenv("SHUFFLE_CERT_DIR") != "" {
for _, file := range files {
if !file.IsDir() {
certPath := filepath.Join(certDir, file.Name())
caCert, err := os.ReadFile(certPath)
if err != nil {
log.Printf("[ERROR] Error reading the certificate %s: %s", file.Name(), err)
} else {
if ok := rootCAs.AppendCertsFromPEM(caCert); ok {
log.Printf("[INFO] Successfully appended certificate: %s", file.Name())
}
}
}
}
transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs}
}
}
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && (strings.ToLower(httpProxy) != "noproxy" || strings.ToLower(httpsProxy) != "noproxy") {
if len(httpProxy) > 0 && strings.ToLower(httpProxy) != "noproxy" {
log.Printf("[DEBUG] Running with HTTP proxy %s (env: HTTP_PROXY). URL: %s", httpProxy, baseUrl)
url_i := url.URL{}
url_proxy, err := url_i.Parse(httpProxy)
if err == nil {
transport.Proxy = http.ProxyURL(url_proxy)
}
}
if len(httpsProxy) > 0 && strings.ToLower(httpsProxy) != "noproxy" {
log.Printf("[DEBUG] Running with HTTPS proxy %s (env: HTTPS_PROXY). URL: %s", httpsProxy, baseUrl)
url_i := url.URL{}
url_proxy, err := url_i.Parse(httpsProxy)
if err == nil {
transport.Proxy = http.ProxyURL(url_proxy)
}
}
}
client := &http.Client{
Transport: transport,
Timeout: time.Second * 60,
}
return client
}
// Function with the name RemoveFromArray to remove a string from a string array
func RemoveFromArray(array []string, element string) []string {
for i, v := range array {
if v == element {
return append(array[:i], array[i+1:]...)
}
}
return array
}
func FindRelevantApps(appname string, apps []WorkflowApp) []WorkflowApp {
return []WorkflowApp{}
}
func FindMatchingCategoryApps(category string, apps []WorkflowApp, org *Org) []WorkflowApp {
if len(category) == 0 {
return []WorkflowApp{}
}
category = strings.ToLower(category)
parsedCategories := map[string]Category{
"siem": org.SecurityFramework.SIEM,
"email": org.SecurityFramework.Communication,
"communication": org.SecurityFramework.Communication,
"assets": org.SecurityFramework.Assets,
"cases": org.SecurityFramework.Cases,
"network": org.SecurityFramework.Network,
"intel": org.SecurityFramework.Intel,
"eradication": org.SecurityFramework.EDR,
"edr": org.SecurityFramework.EDR,
"iam": org.SecurityFramework.IAM,
}
var matchingApps []WorkflowApp
foundCategory, ok := parsedCategories[category]
if ok && len(foundCategory.Name) > 0 {
parsedCategoryNames := strings.Split(foundCategory.Name, ",")
for _, catApp := range parsedCategoryNames {
catApp = strings.ToLower(strings.TrimSpace(catApp))
found := false
for _, app := range apps {
if strings.ToLower(app.Name) == catApp {
matchingApps = append(matchingApps, app)
found = true
break
}
}
if !found {
log.Printf("[INFO] Could not find app %s in category %s", catApp, category)
}
}
}
log.Printf("[INFO] Found %d apps in category '%s'", len(matchingApps), category)
if category == "email" {
category = "communication"
}
for _, app := range apps {
if len(app.Categories) == 0 {
continue
}
if strings.ToLower(app.Categories[0]) != category {
continue
}
matchingApps = append(matchingApps, app)
}
return matchingApps
}
func GetActiveCategories(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed GET category actions: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed authentication"}`))
return
}
ctx := GetContext(request)
newapps, err := GetPrioritizedApps(ctx, user)
if err != nil {
log.Printf("[WARNING] Failed getting apps in category action: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed loading apps. Contact support@shuffler.io"}`))
return
}
categories := GetAllAppCategories()
//AppCategory{
// Name: "Cases",
// Color: "",
// Icon: "cases",
// ActionLabels: []string{"Create ticket"},
// LabelApps: []AppCategoryLabel{
// AppCategoryLabel{
// Name: "Create ticket",
// Apps: []WorkflowApp{
// },
// }
// }
//},
// This is not fast, but ok with just a few hundred thousand iterations :>
log.Printf("[INFO] Starting mapping of labels from all %d apps", len(newapps))
for categoryIndex, _ := range categories {
categories[categoryIndex].AppLabels = []AppLabel{}
}
/*
for labelIndex, label := range category.ActionLabels {
//categories[categoryIndex].LabelApps = append(categories[categoryIndex].LabelApps, AppCategoryLabel{
// Label: label,
// FormattedLabel: strings.ReplaceAll(strings.ToLower(label), " ", "_"),
// Apps: []WorkflowApp{},
//})
}
}
*/
for _, app := range newapps {
if len(app.Name) == 0 {
continue
}
if len(app.Categories) == 0 {
//log.Printf("[INFO] No categories: %#v (%s)", app.Name, app.ID)
continue
}
appLabels := []string{}
for _, action := range app.Actions {
// Compare with formatted label
if len(action.CategoryLabel) > 0 {
//&& strings.ReplaceAll(strings.ToLower(action.CategoryLabel[0]), " ", "_") == categories[categoryIndex].LabelApps[labelIndex].FormattedLabel {
appLabels = append(appLabels, action.CategoryLabel[0])
//AppLabels []AppCategoryLabel `json:"app_labels"`
}
}
if len(appLabels) > 0 {
log.Printf("[DEBUG] '%s' Got labels (%s): %#v", app.Name, app.Categories[0], appLabels)
for categoryIndex, category := range categories {
if strings.ToLower(category.Name) == strings.ToLower(app.Categories[0]) {
newApp := AppLabel{
AppName: app.Name,
LargeImage: app.LargeImage,
ID: app.ID,
}
// FIXME: May need to set the label to be the correct name according to the category's label
for _, appLabel := range appLabels {
newApp.Labels = append(newApp.Labels, LabelStruct{
Category: category.Name,
Label: appLabel,
})
}
categories[categoryIndex].AppLabels = append(categories[categoryIndex].AppLabels, newApp)
}
}
}
}
newjson, err := json.Marshal(categories)
if err != nil {
log.Printf("[WARNING] Failed marshal in get categories: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking categories. Please try again."}`)))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func HandleRecommendationAction(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in modify recommendation: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
http.Error(resp, "Error reading request body", http.StatusInternalServerError)
return
}
var recommendation RecommendationAction
err = json.Unmarshal(body, &recommendation)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling recommendation: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
availableActions := []string{"dismiss"}
if !ArrayContains(availableActions, recommendation.Action) {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Invalid action"}`))
return
}
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting your org details"}`))
return
}
changed := false
for prioIndex, prio := range org.Priorities {
if !prio.Active {
continue
}
if prio.Name != recommendation.Name {
//log.Printf("[DEBUG] '%s' is not '%s'", prio.Name, recommendation.Name)
continue
}
// dismiss first, other later :)
if recommendation.Action == "dismiss" {
org.Priorities[prioIndex].Active = false
changed = true
break
}
}
if changed {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed updating org during priority updates: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": true}`))
return
}
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
// Hard-coded to test out how we can generate next steps in workflows
// This could actually work when mapped back to usecases & with LLMs
// Mainly tested with Outlook Office365 for now
// Should be made based on:
// - Usecases and their structure
// - Active Apps (framework~)
// - LLMs
func HandleActionRecommendation(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Disabled until it gets improved enough to work onprem
// Should be automatically built into the dockerfile of the backend onprem
// Point with cloud download it to have it regularly updated
if project.Environment != "cloud" {
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "Not yet enabled. Contact support@shuffler.io to learn more about progress on this API."}`))
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in get action recommendations: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Get the users' org
ctx := GetContext(request)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting your org details"}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
http.Error(resp, "Error reading request body", http.StatusInternalServerError)
return
}
var workflow Workflow
workflowerr := json.Unmarshal(body, &workflow)
if workflowerr != nil {
log.Printf("[WARNING] Failed unmarshalling workflow: %s", workflowerr)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
// Need apps to check against
// These are also the only ones we're able to recommend from
// Using the app framework + these we can generate recommendations :)
apps, err := GetPrioritizedApps(ctx, user)
if err != nil {
log.Printf("[WARNING] Failed getting apps during node suggestion validation: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to fetch recommendation data"}`))
return
}
// Load in node relations
nodeRelations, err := GetNodeRelations(ctx)
if err != nil {
log.Printf("[WARNING] Failed getting node relations: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false, "reason": "Failed to fetch recommendation data"}`))
return
}
var recommendAction ActionRecommendations
if len(workflow.Actions) == 0 {
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "No actions in workflow"}`))
return
}
// Because this usually means "formatting"
skippable := []string{"repeat_back_to_me"}
// More testing based on node relation output
// Goal with this is to test singular node connections
// Next step: Use multi-step checks to improve further
for key, inputAction := range workflow.Actions {
var recommendations []Recommendations
var action RecommendAction
app := workflow.Actions[key].AppName + "_" + workflow.Actions[key].AppVersion
_ = app
// Check if there is any label for the current action we're using
//log.Printf("App: %s", app)
foundApp := WorkflowApp{}
if len(inputAction.CategoryLabel) == 0 {
for _, app := range apps {
if app.Name != inputAction.AppName && app.ID != inputAction.AppID {
continue
}
foundApp = app
if foundApp.Name == "Shuffle Tools" {
inputAction.CategoryLabel = []string{inputAction.Name}
break
}
// Look for the correct action
//log.Printf("Found app %s", app.Name)
for _, action := range app.Actions {
if action.Name == inputAction.Name {
//log.Printf("Found action %s", action.Name)
if len(action.CategoryLabel) > 0 {
inputAction.CategoryLabel = action.CategoryLabel
break
}
}
}
break
}
}
if len(inputAction.CategoryLabel) == 0 {
//log.Printf("No labels for action %s in app %s", inputAction.Name, inputAction.AppName)
continue
}
//log.Printf("Action %s (%s) has %d labels: %#v", inputAction.Name,foundApp.Name, len(inputAction.CategoryLabel), inputAction.CategoryLabel)
parsedCategory := strings.ToLower(strings.Replace(inputAction.CategoryLabel[0], " ", "_", -1))
// Check synonyms
for key, node := range nodeRelations {
if len(node.Synonyms) == 0 {
continue
}
for _, synonym := range node.Synonyms {
if synonym == parsedCategory {
log.Printf("[DEBUG] Found new synonym '%s' for '%s'", synonym, parsedCategory)
parsedCategory = key
break
}
}
if parsedCategory == key {
break
}
}
// Specific parsing
if parsedCategory == "repeat_back_to_me" {
continue
}
//log.Printf("Looking for category: %s", parsedCategory)
for category, categoryValue := range nodeRelations {
//log.Printf("Checking category %s vs %s", category, parsedCategory)
if category != parsedCategory {
continue
}
// Choose first 2 outgoing nodes
for cnt, outgoing := range categoryValue.Outgoing {
//log.Printf("Found outgoing %s:%d", outgoing.Name, outgoing.Count)
categoryname := outgoing.Name
if ArrayContains(skippable, categoryname) {
continue
}
// Check if categoryname in nodeRelations map
foundAppType := ""
if foundWrapper, ok := nodeRelations[categoryname]; ok {
foundAppType = foundWrapper.AppCategory
} else {
log.Printf("No node relations for %s", categoryname)
continue
}
recommendation := Recommendations{}
if foundAppType == "tools" {
recommendation = Recommendations{
AppName: "Shuffle Tools",
AppAction: outgoing.Name,
AppVersion: "1.2.0",
AppId: "bc78f35c6c6351b07a09b7aed5d29652",
}
} else if categoryname == "subflow" {
recommendation = Recommendations{
AppName: "Shuffle Subflow",
AppVersion: "1.1.0",
AppAction: "subflow",
AppId: "a891257fcf905c2d256ce5674282864c",
}
} else {
log.Printf("[DEBUG] Found app category %s for category %s", foundAppType, categoryname)
foundCategory := Category{}
if foundAppType == "cases" {
foundCategory = org.SecurityFramework.Cases
} else if foundAppType == "communication" {
foundCategory = org.SecurityFramework.Communication
} else if foundAppType == "assets" {
foundCategory = org.SecurityFramework.Assets
} else if foundAppType == "network" {
foundCategory = org.SecurityFramework.Network
} else if foundAppType == "intel" {
foundCategory = org.SecurityFramework.Intel
} else if foundAppType == "edr" {
foundCategory = org.SecurityFramework.EDR
} else if foundAppType == "iam" {
foundCategory = org.SecurityFramework.IAM
} else if foundAppType == "siem" {
foundCategory = org.SecurityFramework.SIEM
} else {
//foundCategory = org.SecurityFramework.Other
}
if foundCategory.Name == "" {
log.Printf("[ERROR] No app found for category %s", categoryname)
continue
}
// TODO: Find the name of the action in the app that has the category label
foundAction := WorkflowAppAction{}
for _, action := range foundApp.Actions {
if len(action.CategoryLabel) == 0 {
continue
}
// make all labels lowercase and with underscore
newLabels := []string{}
for _, label := range action.CategoryLabel {
newLabels = append(newLabels, strings.ToLower(strings.Replace(label, " ", "_", -1)))
}
if ArrayContains(newLabels, categoryname) {
foundAction = action
break
}
}
if foundAction.Name == "" {
log.Printf("[ERROR] No action explainer found for category '%s'", categoryname)
} else {
log.Printf("[DEBUG] Found action %s for category %s", foundAction.Name, categoryname)
}
recommendation = Recommendations{
AppName: foundCategory.Name,
AppId: foundCategory.ID,
AppAction: foundAction.Name,
}
}
if recommendation.AppName != "" {
recommendations = append(recommendations, recommendation)
}
if cnt == 1 {
break
}
}
break
}
//log.Printf("[DEBUG] Found %d recommendations for action %s", len(recommendations), inputAction.Name)
action.ActionId = inputAction.ID
action.AppName = inputAction.AppName
action.Recommendations = recommendations
recommendAction.Actions = append(recommendAction.Actions, action)
}
recommendAction.Success = true
newjson, err := json.Marshal(recommendAction)
if err != nil {
log.Printf("[ERROR] Failed to marshal recommendedAction: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(newjson)
}
func HandleGetenvStats(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get env stats executions: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin during stop executions")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
ctx := GetContext(request)
environmentName := fileId
if len(fileId) != 36 {
log.Printf("[DEBUG] Environment length %d for %s is not good for env Stats. Attempting to find the actual ID for it", len(fileId), fileId)
environments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting environments to validate: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed to validate environment"}`))
return
}
for _, environment := range environments {
if environment.Name == fileId && len(environment.Id) > 0 {
environmentName = fileId
fileId = environment.Id
break
}
}
if len(fileId) != 36 {
log.Printf("[WARNING] Failed getting environments to validate. New FileId: %s", fileId)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting environment for ID %s"}`, fileId)))
return
}
}
// Should get stats for this
_ = environmentName
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func HandleSetenvConfig(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in set env config: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUDIT] User isn't admin during set env config")
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Must be admin to perform this action"}`)))
return
}
ctx := GetContext(request)
var environmentId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
environmentId = location[4]
}
if len(environmentId) == 0 {
log.Printf("[Error] No environment ID found in path")
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
type environmentConfig struct {
Action string `json:"action"`
SelectedSuborg []string `json:"selected_suborgs"`
}
var config environmentConfig
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[Error] Failed reading body in set env config: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
err = json.Unmarshal(body, &config)
if err != nil {
log.Printf("[Error] Failed unmarshalling body in set env config: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
environment, err := GetEnvironment(ctx, environmentId, user.ActiveOrg.Id)
if err != nil {
log.Printf("[Error] Failed getting environment in set env config: %s for Id: %s", err, environmentId)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
if config.Action == "suborg_distribute" {
if len(config.SelectedSuborg) == 0 {
environment.SuborgDistribution = []string{}
} else {
environment.SuborgDistribution = config.SelectedSuborg
}
err = SetEnvironment(ctx, environment)
if err != nil {
log.Printf("[Error] Failed setting environment in set env config: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
foundOrg, err := GetOrg(ctx, user.ActiveOrg.Id)
if err == nil {
for _, childOrg := range foundOrg.ChildOrgs {
DeleteCache(ctx, fmt.Sprintf("Environments_%s", childOrg.Id))
}
}
log.Printf("[INFO] Successfully updated environment in set env config for environment id: %s", environmentId)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason" : "Successfully updated environment"}`))
return
}
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Invalid action"}`))
}
func GetWorkflowSuggestions(ctx context.Context, user User, org *Org, orgUpdated bool, amount int) (*Org, bool) {
// Loop workflows
//if amount > 3 {
// log.Printf("[WARNING] Amount of suggestions is too high: %d", amount)
// return org, orgUpdated
//}
// 1. Suggest based on usecases
// 2. Suggest public workflows (cloud)
// 3. Use workflow template (local)
var updated bool
workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "")
if err != nil {
log.Printf("[WARNING] No workflows found for user %s (2)", user.Id)
return org, orgUpdated
}
//log.Printf("[INFO] Finding workflow suggestions for %s (%s) based on %d workflows", org.Name, org.Id, len(workflows))
for _, workflow := range workflows {
for _, action := range workflow.Actions {
if len(action.Category) == 0 {
continue
}
if org.SecurityFramework.Communication.Name == "" && (action.Category == "Communication" || action.Category == "email") {
orgUpdated = true
org.SecurityFramework.Communication = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.Intel.Name == "" && action.Category == "Intel" {
orgUpdated = true
org.SecurityFramework.Intel = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.Network.Name == "" && action.Category == "Network" {
orgUpdated = true
org.SecurityFramework.Network = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.Assets.Name == "" && action.Category == "Assets" {
orgUpdated = true
org.SecurityFramework.Assets = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.Cases.Name == "" && action.Category == "Cases" {
orgUpdated = true
org.SecurityFramework.Cases = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.SIEM.Name == "" && action.Category == "SIEM" {
orgUpdated = true
org.SecurityFramework.SIEM = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.EDR.Name == "" && action.Category == "EDR" {
orgUpdated = true
org.SecurityFramework.EDR = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
if org.SecurityFramework.IAM.Name == "" && action.Category == "IAM" {
orgUpdated = true
org.SecurityFramework.IAM = Category{
Name: action.AppName,
Count: 1,
Description: "",
LargeImage: action.LargeImage,
ID: action.AppID,
}
}
}
}
// Checking again to see if specifying either should be a priority
missingType := ""
amountDone := 0
if missingType == "" && org.SecurityFramework.SIEM.Name == "" {
missingType = "SIEM"
amountDone = 1
} else if missingType == "" && org.SecurityFramework.Communication.Name == "" {
missingType = "Email"
amountDone = 2
} else if missingType == "" && org.SecurityFramework.EDR.Name == "" {
missingType = "EDR"
amountDone = 3
} else if missingType == "" && org.SecurityFramework.Cases.Name == "" {
missingType = "Cases"
amountDone = 4
} else if missingType == "" && org.SecurityFramework.Intel.Name == "" {
missingType = "Intel"
amountDone = 5
} else if missingType == "" && org.SecurityFramework.Network.Name == "" {
missingType = "Network"
amountDone = 6
} else if missingType == "" && org.SecurityFramework.Assets.Name == "" {
missingType = "Assets"
amountDone = 7
} else if missingType == "" && org.SecurityFramework.IAM.Name == "" {
missingType = "IAM"
amountDone = 8
}
if len(missingType) > 0 {
org, updated = AddPriority(*org, Priority{
Name: fmt.Sprintf("Your Organizations' %s App hasn't been specified (%d/8)", missingType, amountDone),
Description: fmt.Sprintf("Your %s system should be specified to enable us to suggest relevant usecases to you", missingType),
Type: "apps",
Active: true,
URL: fmt.Sprintf("/welcome?tab=2&target=%s", missingType),
Severity: 3,
}, updated)
if updated {
orgUpdated = true
}
}
//org.SecurityFramework.EDR.Name == "" || org.SecurityFramework.Communication.Name == "" {
// Checking which workflows SHOULD have a usecase attached to them
for _, workflow := range workflows {
if len(workflow.UsecaseIds) != 0 {
continue
}
//log.Printf("[INFO] No usecase for workflow %s", workflow.Name)
// Sample: If email (get/trigger) & cases (create ticket) in same workflow -> email usecase = done
// If excel/sheets is used, reporting
// Add keywords to usecases? Check if anything matching in:
// - name
// - action name
// - action label(s)
// - action description
}
// Matching org priority with usecases & previously built workflows
usecasesAdded := 0
for _, orgPriority := range org.Priorities {
if orgPriority.Type != "usecase" || !orgPriority.Active {
continue
}
usecasesAdded += 1
}
var usecases UsecaseLinks
err = json.Unmarshal([]byte(GetUsecaseData()), &usecases)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal usecase data (priorities): %s", err)
} else {
//log.Printf("[DEBUG] Got parsed usecases for %s - should check priority vs mainpriority (%s)", org.Name, org.MainPriority)
selectedAppName := ""
selectedAppImage := ""
innerUpdate := false
for usecaseIndex, usecase := range usecases {
//if usecase.Name != org.MainPriority {
// continue
//}
// match them with usecases here
for _, workflow := range workflows {
if len(workflow.UsecaseIds) == 0 {
continue
}
// Fidning matching usecase for workflow
for _, workflowUsecase := range workflow.UsecaseIds {
newUsecasename := strings.ToLower(workflowUsecase)
for subusecaseIndex, subusecase := range usecase.List {
if newUsecasename == strings.ToLower(subusecase.Name) {
usecases[usecaseIndex].List[subusecaseIndex].Matches = append(usecases[usecaseIndex].List[subusecaseIndex].Matches, workflow)
break
}
}
}
}
// Sort sub-usecases by priority
slice.Sort(usecase.List[:], func(i, j int) bool {
return usecase.List[i].Priority > usecase.List[j].Priority
})
//log.Printf("[DEBUG] Priorities for %s", usecase.Name)
cntAdded := 0
for _, subusecase := range usecase.List {
// Matches = matching usecases that have workflows attached to them
// This means if it exists, don't add it as priority
if len(subusecase.Matches) > 0 {
continue
}
if strings.ToLower(subusecase.Type) == "iam" {
if org.SecurityFramework.IAM.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.IAM.Name
selectedAppImage = org.SecurityFramework.IAM.LargeImage
}
if strings.ToLower(subusecase.Type) == "siem" {
if org.SecurityFramework.SIEM.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.SIEM.Name
selectedAppImage = org.SecurityFramework.SIEM.LargeImage
}
if strings.ToLower(subusecase.Type) == "edr" {
if org.SecurityFramework.EDR.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.EDR.Name
selectedAppImage = org.SecurityFramework.EDR.LargeImage
}
if strings.ToLower(subusecase.Type) == "communication" {
if org.SecurityFramework.Communication.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.Communication.Name
selectedAppImage = org.SecurityFramework.Communication.LargeImage
}
if strings.ToLower(subusecase.Type) == "assets" {
if org.SecurityFramework.Assets.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.Assets.Name
selectedAppImage = org.SecurityFramework.Assets.LargeImage
}
if strings.ToLower(subusecase.Type) == "cases" {
if org.SecurityFramework.Cases.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.Cases.Name
selectedAppImage = org.SecurityFramework.Cases.LargeImage
}
if strings.ToLower(subusecase.Type) == "network" {
if org.SecurityFramework.Network.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.Network.Name
selectedAppImage = org.SecurityFramework.Network.LargeImage
}
if strings.ToLower(subusecase.Type) == "intel" {
if org.SecurityFramework.Intel.Name == "" {
continue
}
selectedAppName = org.SecurityFramework.Intel.Name
selectedAppImage = org.SecurityFramework.Intel.LargeImage
}
usecaseDescription := "A priority usecase for your organization has been found. Click explore to learn more."
if len(selectedAppName) > 0 && len(selectedAppImage) > 0 && subusecase.Type != subusecase.Last {
usecaseDescription = fmt.Sprintf("%s&%s", strings.Replace(selectedAppName, "_", " ", -1), selectedAppImage)
// Adding "last" node as well
if strings.ToLower(subusecase.Last) == "iam" && org.SecurityFramework.IAM.Name != "" && org.SecurityFramework.IAM.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.IAM.Name, "_", " ", -1), org.SecurityFramework.IAM.LargeImage)
} else if strings.ToLower(subusecase.Last) == "siem" && org.SecurityFramework.SIEM.Name != "" && org.SecurityFramework.SIEM.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.SIEM.Name, "_", " ", -1), org.SecurityFramework.SIEM.LargeImage)
} else if strings.ToLower(subusecase.Last) == "edr" && org.SecurityFramework.EDR.Name != "" && org.SecurityFramework.EDR.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.EDR.Name, "_", " ", -1), org.SecurityFramework.EDR.LargeImage)
} else if strings.ToLower(subusecase.Last) == "communication" && org.SecurityFramework.Communication.Name != "" && org.SecurityFramework.Communication.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.Communication.Name, "_", " ", -1), org.SecurityFramework.Communication.LargeImage)
} else if strings.ToLower(subusecase.Last) == "assets" && org.SecurityFramework.Assets.Name != "" && org.SecurityFramework.Assets.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.Assets.Name, "_", " ", -1), org.SecurityFramework.Assets.LargeImage)
} else if strings.ToLower(subusecase.Last) == "cases" && org.SecurityFramework.Cases.Name != "" && org.SecurityFramework.Cases.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.Cases.Name, "_", " ", -1), org.SecurityFramework.Cases.LargeImage)
} else if strings.ToLower(subusecase.Last) == "network" && org.SecurityFramework.Network.Name != "" && org.SecurityFramework.Network.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.Network.Name, "_", " ", -1), org.SecurityFramework.Network.LargeImage)
} else if strings.ToLower(subusecase.Last) == "intel" && org.SecurityFramework.Intel.Name != "" && org.SecurityFramework.Intel.LargeImage != "" {
usecaseDescription = fmt.Sprintf("%s&%s&%s", usecaseDescription, strings.Replace(org.SecurityFramework.Intel.Name, "_", " ", -1), org.SecurityFramework.Intel.LargeImage)
}
} else if len(subusecase.Last) > 0 && subusecase.Type == subusecase.Last {
usecaseDescription = fmt.Sprintf("%s&%s&%s:default&", strings.Replace(selectedAppName, "_", " ", -1), selectedAppImage, subusecase.Last)
}
usecaseDescription += "&" + subusecase.Description
// Should find info about the usecase
// No description as this has custom rendering
org, innerUpdate = AddPriority(*org, Priority{
Name: fmt.Sprintf("Suggested Usecase: %s", subusecase.Name),
Description: usecaseDescription,
Type: "usecase",
Active: true,
URL: fmt.Sprintf("/usecases?selected_object=%s", subusecase.Name),
Severity: 3,
}, updated)
if innerUpdate {
//log.Printf("[DEBUG] Org %s (%s) got the priority for Usecase '%s' added. Added: %d", org.Name, org.Id, subusecase.Name, usecasesAdded)
cntAdded += 1
orgUpdated = true
usecasesAdded += 1
if usecasesAdded >= 3 {
break
}
}
}
if innerUpdate && usecasesAdded >= 3 {
break
}
}
}
if usecasesAdded < 3 {
//log.Printf("[DEBUG] Should check if workflows still are the same amount or not to change priorities")
// Check all existing priorities if they should still be closed, or reopened
for prioIndex, priority := range org.Priorities {
if priority.Type != "usecase" || priority.Active == true {
continue
}
// Check if the usecase is still in the workflow list
usecaseName := strings.ReplaceAll(priority.Name, "Suggested Usecase: ", "")
found := false
for _, workflow := range workflows {
for _, usecase := range workflow.UsecaseIds {
if usecase == usecaseName {
found = true
break
}
}
if found {
break
}
}
if !found {
if usecasesAdded < 3 {
usecasesAdded += 1
orgUpdated = true
org.Priorities[prioIndex].Active = true
} else {
break
}
}
}
}
//if usecasesAdded <= 3 {
// return GetWorkflowSuggestions(ctx, user, org, orgUpdated, amount+1)
//}
if usecasesAdded < 3 {
//log.Printf("\n\n[DEBUG] Should generate priorities for org %s (%s) based on purely numbers\n\n", org.Name, org.Id)
newPrios := []Priority{
Priority{
Name: "Suggested Usecase: SIEM to ticket",
Description: "siem:default&&cases:default&&SIEM to ticket is a usecase that is very common in most organizations. It is a usecase that is very important to get right, as it is the most common way for attackers to get into your organization.",
Type: "usecase",
Active: true,
URL: "/usecases?selected_object=SIEM to ticket",
Severity: 3,
}, Priority{
Name: "Suggested Usecase: Email management",
Description: "edr:default&&cases:default&&Email management is a usecase that is very common in most organizations. It is a usecase that is very important to get right, as it is the most common way for attackers to get into your organization.",
Type: "usecase",
Active: true,
URL: "/usecases?selected_object=Email management",
Severity: 3,
}, Priority{
Name: "Suggested Usecase: EDR to ticket",
Description: "communication:default&&cases:default&&EDR to ticket is a usecase that is very common in most organizations. It is a usecase that is very important to get right, as it is the most common way for attackers to get into your organization.",
Type: "usecase",
Active: true,
URL: "/usecases?selected_object=EDR to ticket",
Severity: 3,
},
}
for _, prio := range newPrios {
prioName := strings.ToLower(strings.ReplaceAll(prio.Name, "Suggested Usecase: ", ""))
found := false
for _, existingPrio := range org.Priorities {
if strings.Contains(strings.ToLower(strings.ReplaceAll(existingPrio.Name, "Suggested Usecase: ", "")), prioName) {
//log.Printf("[DEBUG] Org %s (%s) already has the priority for Usecase '%s' added. Added: %d", org.Name, org.Id, prio.Name, usecasesAdded)
found = true
break
}
}
if !found {
org.Priorities = append(org.Priorities, prio)
usecasesAdded += 1
// Force not org update due to this being temporary
orgUpdated = false
if usecasesAdded >= 3 {
break
}
}
}
} else {
//log.Printf("[DEBUG] Org %s (%s) already has the priorities added. Added: %d", org.Name, org.Id, usecasesAdded)
}
return org, orgUpdated
}
func GetDatastoreKeyRevisions(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Removed check here as it may be a public workflow
user, err := HandleApiAuthentication(resp, request)
if err != nil {
//log.Printf("[AUDIT] Api authentication failed in getting workflow revisions: %s. Continuing because it may be public.", err)
log.Printf("[AUDIT] Api authentication failed in getting workflow revisions: %s. ", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var category string
var key string
if location[1] == "api" {
if len(location) <= 6 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
category = location[5]
key = location[6]
}
if len(category) == 0 || len(key) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Category or Key is empty"}`))
return
}
ctx := GetContext(request)
datastoreKeys, err := GetDatastoreRevisions(ctx, key, category, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed loading key revisions for %s (%s).", key, category)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed finding workflow"}`))
return
}
parsedKeys := []CacheKeyData{}
toDelete := []CacheKeyData{}
cutoff := time.Now().AddDate(0, 0, -30)
for _, key := range datastoreKeys {
if key.OrgId != user.ActiveOrg.Id {
continue
}
if key.Category != category {
continue
}
editedTime := time.Unix(key.Edited, 0)
if editedTime.Before(cutoff) {
toDelete = append(toDelete, key)
continue
}
parsedKeys = append(parsedKeys, key)
}
if len(toDelete) > 0 && debug {
log.Printf("[DEBUG] Deleting %d old datastore revisions\n\n", len(toDelete))
}
nameKey := "org_cache_revisions"
for _, cacheData := range toDelete {
cacheData.RevisionId = uuid.NewV4().String()
cacheId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
if len(cacheData.Category) > 0 && cacheData.Category != "default" {
cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.Category)
}
cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.RevisionId)
// URL encode
cacheId = url.QueryEscape(cacheId)
if len(cacheId) > 127 {
cacheId = cacheId[:127]
}
go DeleteKey(context.Background(), nameKey, cacheId)
}
body, err := json.Marshal(parsedKeys)
if err != nil {
log.Printf("[WARNING] Failed datastore key revision GET marshalling: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(body)
}
func GetWorkflowRevisions(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Removed check here as it may be a public workflow
user, err := HandleApiAuthentication(resp, request)
if err != nil {
//log.Printf("[AUDIT] Api authentication failed in getting workflow revisions: %s. Continuing because it may be public.", err)
log.Printf("[AUDIT] Api authentication failed in getting workflow revisions: %s. ", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`))
return
}
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Workflow %s doesn't exist.", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed finding workflow"}`))
return
}
// Check workflow.Sharing == private / public / org too
if user.Id != workflow.Owner || len(user.Id) == 0 {
// Added org-reader as the user should be able to read everything in an org
//if workflow.OrgId == user.ActiveOrg.Id && (user.Role == "admin" || user.Role == "org-reader") {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (get workflow revisions)", user.Username, workflow.ID)
// Only for Read-Only. No executions or impersonations.
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow revisions for %s", user.Username, fileId)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (get workflow revisions). Verified: %t, Active: %t, SupportAccess: %t, Username: %s", user.Username, workflow.ID, user.Verified, user.Active, user.SupportAccess, user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
revisionCount := 50
if request.URL.Query().Get("count") != "" {
revisionCount, err = strconv.Atoi(request.URL.Query().Get("count"))
if err != nil {
log.Printf("[WARNING] Failed converting count to int: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed converting count to int"}`))
return
}
}
// Access is granted -> get revisions
revisions, err := ListWorkflowRevisions(ctx, workflow.ID, revisionCount)
if err != nil {
log.Printf("[WARNING] Failed getting revisions for workflow %s: %s", workflow.ID, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := json.Marshal(revisions)
if err != nil {
log.Printf("[WARNING] Failed workflow GET marshalling: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(body)
}
func HandleDeleteOrg(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Checking if it's a special region. All user-specific requests should
// go through shuffler.io and not subdomains
if project.Environment == "cloud" {
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting DELETE ORG request to main site handler (shuffler.io)")
RedirectUserRequest(resp, request)
return
}
}
var fileId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
ctx := GetContext(request)
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in DELETING specific org: %s. Continuing because it may be public.", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[WARNING] Not admin: %s (%s).", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Not admin"}`))
return
}
if user.SessionLogin {
// get the request body
type ReturnData struct {
Password string `json:"password"`
}
var tmpData ReturnData
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed reading body in delete org: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
}
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling body in delete org: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed unmarshalling body"}`))
return
}
// check if the password is correct
if len(tmpData.Password) == 0 {
log.Printf("[WARNING] No password provided in delete org request")
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No password provided"}`))
return
}
err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(tmpData.Password))
if err != nil {
log.Printf("[WARNING] Password for user %s is incorrect in delete org request", user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Incorrect password"}`))
return
}
}
subOrg, err := GetOrg(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", fileId, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org details"}`))
return
}
if len(subOrg.CreatorOrg) == 0 {
log.Printf("[WARNING] Org '%s' has no parent org. Not deleting.", fileId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Org is not a child org of the parent org"}`))
return
}
parentOrg, err := GetOrg(ctx, subOrg.CreatorOrg)
if err != nil {
log.Printf("[WARNING] Failed getting org '%s': %s", fileId, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org details"}`))
return
}
isAdmin := false
for _, orgUser := range parentOrg.Users {
if orgUser.Username == user.Username && orgUser.Role == "admin" {
isAdmin = true
break
}
}
if !isAdmin && !user.SupportAccess {
log.Printf("[WARNING] User %s is not an admin in org '%s'. Not deleting.", user.Username, fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "User is not an admin in the parent org"}`))
return
}
// Get workflows
currentActiveOrg := user.ActiveOrg
user.ActiveOrg.Id = subOrg.Id
user.ActiveOrg.Name = subOrg.Name
workflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "")
if err != nil {
log.Printf("[WARNING] Failed getting workflows for user %s (0): %s", user.Username, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting workflows"}`))
return
}
// Return if workflows, as they should be deleted beforehand
if len(workflows) > 0 {
log.Printf("[WARNING] Org '%s' has %d workflow(s). Not deleting.", fileId, len(workflows))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Org still has workflows. Delete them first by listing the /api/v1/workflows API."}`))
return
}
// Delete the org
err = DeleteKey(ctx, "Organizations", subOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed deleting org '%s': %s", subOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed deleting org"}`))
return
}
newOrgString := []string{}
for _, orgId := range user.Orgs {
if orgId != subOrg.Id {
newOrgString = append(newOrgString, orgId)
}
}
newChildOrg := []OrgMini{}
for _, childOrg := range parentOrg.ChildOrgs {
if childOrg.Id != subOrg.Id {
newChildOrg = append(newChildOrg, childOrg)
}
}
parentOrg.ChildOrgs = newChildOrg
suborgCacheKey := fmt.Sprintf("%s__childorgs", parentOrg.Id)
DeleteCache(ctx, suborgCacheKey)
DeleteCache(ctx, fmt.Sprintf("Organizations_%s", subOrg.Id))
parentOrg.SyncUsage.MultiTenant.Counter = int64(len(newChildOrg)) + 1
parentOrg.SyncFeatures.MultiTenant.Usage = int64(len(newChildOrg)) + 1
err = SetOrg(ctx, *parentOrg, parentOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed setting parent org '%s': %s", parentOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting parent org"}`))
return
}
user.Orgs = newOrgString
if user.ActiveOrg.Id == subOrg.Id {
// If the user is in the org that was deleted, set active org as parent org
user.ActiveOrg.Id = parentOrg.Id
user.ActiveOrg.Name = parentOrg.Name
} else {
user.ActiveOrg.Id = currentActiveOrg.Id
user.ActiveOrg.Name = currentActiveOrg.Name
}
err = SetUser(ctx, &user, true)
if err != nil {
log.Printf("[WARNING] Failed setting user '%s': %s", user.Username, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting user"}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
}
func AssignAuthEverywhere(ctx context.Context, auth *AppAuthenticationStorage, user User) error {
log.Printf("[INFO] Should set authentication config")
baseWorkflows, err := GetAllWorkflowsByQuery(ctx, user, 250, "")
if err != nil && len(baseWorkflows) == 0 {
log.Printf("Getall error in auth update: %s", err)
return err
}
workflows := []Workflow{}
for _, workflow := range baseWorkflows {
if workflow.OrgId == user.ActiveOrg.Id {
workflows = append(workflows, workflow)
}
}
// FIXME: Add function to remove auth from other auth's
actionCnt := 0
workflowCnt := 0
authenticationUsage := []AuthenticationUsage{}
for _, workflow := range workflows {
newActions := []Action{}
edited := false
usage := AuthenticationUsage{
WorkflowId: workflow.ID,
Nodes: []string{},
}
for _, action := range workflow.Actions {
if action.AppName == auth.App.Name {
action.AuthenticationId = auth.Id
edited = true
actionCnt += 1
usage.Nodes = append(usage.Nodes, action.ID)
}
newActions = append(newActions, action)
}
workflow.Actions = newActions
if edited {
//auth.Usage = usage
authenticationUsage = append(authenticationUsage, usage)
err = SetWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("Failed setting (authupdate) workflow: %s", err)
continue
}
//cacheKey := fmt.Sprintf("%s_workflows", user.Id)
//DeleteCache(ctx, cacheKey)
workflowCnt += 1
}
}
//Usage []AuthenticationUsage `json:"usage" datastore:"usage"`
log.Printf("[INFO] Found %d workflows, %d actions", workflowCnt, actionCnt)
if actionCnt > 0 && workflowCnt > 0 {
auth.WorkflowCount = int64(workflowCnt)
auth.NodeCount = int64(actionCnt)
auth.Usage = authenticationUsage
auth.Defined = true
err = SetWorkflowAppAuthDatastore(ctx, *auth, auth.Id)
if err != nil {
log.Printf("Failed setting appauth: %s", err)
return err
} else {
// FIXME: Remove ALL workflows from other auths using the same
}
}
return nil
}
func HandleWorkflowRunSearch(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[WARNING] Api authentication failed in search workflow runs: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[WARNING] Failed workflow body read (workflow search): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
search := WorkflowSearch{}
err = json.Unmarshal([]byte(body), &search)
if err != nil {
//log.Printf(string(body))
log.Printf("[ERROR] Failed workflow unmarshaling (workflow search): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
//log.Printf("[DEBUG] Got Run search: %+v", search)
// Here to check access rights
ctx := GetContext(request)
if len(search.WorkflowId) > 0 {
workflow, err := GetWorkflow(ctx, search.WorkflowId)
if err != nil {
log.Printf("[WARNING] Failed getting the workflow %s locally (search workflow runs): %s", search.WorkflowId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Check workflow.Sharing == private / public / org too
if user.Id != workflow.Owner || len(user.Id) == 0 {
// Added org-reader as the user should be able to read everything in an org
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (workflow run search)", user.Username, workflow.ID)
} else if workflow.Public {
log.Printf("[AUDIT] Letting user %s access workflow %s because it's public", user.Username, workflow.ID)
// Only for Read-Only. No executions or impersonations.
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow run debug search for %s", user.Username, workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (workflow run search). Verified: %t, Active: %t, SupportAccess: %t, Username: %s", user.Username, workflow.ID, user.Verified, user.Active, user.SupportAccess, user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
}
chosenOrg := user.ActiveOrg.Id
if search.IgnoreOrg == true && user.SupportAccess {
chosenOrg = ""
}
runs, cursor, err := GetWorkflowRunsBySearch(ctx, chosenOrg, search)
if err != nil {
log.Printf("[WARNING] Failed getting workflow runs by search: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
parsedRuns := []WorkflowExecution{}
for _, run := range runs {
if run.ExecutionOrg != user.ActiveOrg.Id {
if !user.SupportAccess {
continue
}
}
parsedRuns = append(parsedRuns, run)
}
runs = parsedRuns
workflowSearchResult := WorkflowSearchResult{
Success: true,
Runs: runs,
Cursor: cursor,
}
//Get workflow run for all subgs of current org where the user is a member
if search.SuborgRuns == true {
suborgs, _, err := GetAllChildOrgs(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[WARNING] Failed getting suborgs for org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
// Limit to max 50 suborgs
if len(suborgs) > 50 {
suborgs = suborgs[:50]
}
type validationResult struct {
org Org
valid bool
}
resultChan := make(chan validationResult, len(suborgs))
var wg sync.WaitGroup
// Validate suborgs concurrently
for _, suborg := range suborgs {
wg.Add(1)
go func(suborg Org) {
defer wg.Done()
userPresentInSuborg := false
for _, orgId := range user.Orgs {
if orgId == suborg.Id || user.SupportAccess == true {
userPresentInSuborg = true
break
}
}
resultChan <- validationResult{
org: suborg,
valid: userPresentInSuborg,
}
}(suborg)
}
// Close channel when all validations complete
go func() {
wg.Wait()
close(resultChan)
}()
// Collect valid suborgs
validSuborgs := []Org{}
for result := range resultChan {
if result.valid {
validSuborgs = append(validSuborgs, result.org)
}
}
type batchResult struct {
runs []WorkflowExecution
err error
}
runsChan := make(chan batchResult, len(validSuborgs))
wg = sync.WaitGroup{}
// Process each valid suborg concurrently
for _, suborg := range validSuborgs {
wg.Add(1)
go func(suborg Org) {
defer wg.Done()
runs, _, err := GetWorkflowRunsBySearch(ctx, suborg.Id, search)
if err != nil {
runsChan <- batchResult{
err: fmt.Errorf("failed getting workflow runs for suborg %s: %v", suborg.Id, err),
}
return
}
// Filter runs and add suborg details
parsedRuns := []WorkflowExecution{}
for _, run := range runs {
run.Org = OrgMini{
Id: suborg.Id,
Name: suborg.Name,
Image: suborg.Image,
CreatorOrg: suborg.CreatorOrg,
RegionUrl: suborg.RegionUrl,
}
parsedRuns = append(parsedRuns, run)
}
runsChan <- batchResult{runs: parsedRuns}
}(suborg)
}
// Close channel when all goroutines complete
go func() {
wg.Wait()
close(runsChan)
}()
// Collect results from all suborgs
suborgRuns := []WorkflowExecution{}
for result := range runsChan {
if result.err != nil {
log.Printf("[WARNING] %v", result.err)
continue
}
suborgRuns = append(suborgRuns, result.runs...)
}
// Combine parent and suborg runs
allRuns := append(workflowSearchResult.Runs, suborgRuns...)
// Sort by start time
sort.Slice(allRuns, func(i, j int) bool {
return allRuns[i].StartedAt > allRuns[j].StartedAt
})
workflowSearchResult.Runs = allRuns
}
respBody, err := json.Marshal(workflowSearchResult)
if err != nil {
log.Printf("[WARNING] Failed marshaling workflow runs: %s", err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
resp.Write(respBody)
}
func LoadUsecases(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get usecases. Continuing anyway: %s", err)
//resp.WriteHeader(401)
//resp.Write([]byte(`{"success": false}`))
//return
}
// FIXME: Load for the specific org and have structs for it all
_ = user
//ctx := GetContext(request)
resp.WriteHeader(200)
resp.Write([]byte(GetUsecaseData()))
}
func parseSubflowResults(ctx context.Context, result ActionResult) (ActionResult, bool) {
var parentSubflowResult []SubflowData
err := json.Unmarshal([]byte(result.Result), &parentSubflowResult)
if err != nil {
//log.Printf("[WARNING] Failed unmarshaling subflow result. This could be due to it not being a list: %s", err)
return result, false
}
for _, param := range result.Action.Parameters {
if param.Name == "check_result" {
if param.Value == "false" {
return result, false
}
}
}
newResults := []SubflowData{}
finishedSubflows := 0
failedCount := 0
for _, res := range parentSubflowResult {
// If value length = 0 for any, then check cache and add the result
if res.ResultSet && len(res.Result) > 0 {
//log.Printf("[DEBUG][%s] Got result set for subflow. Result: %#v", res.ExecutionId, res.Result)
newResults = append(newResults, res)
finishedSubflows += 1
continue
}
if !res.Success {
//log.Printf("[DEBUG][%s] Subflow failed", res.ExecutionId)
newResults = append(newResults, res)
failedCount += 1
finishedSubflows += 1
continue
}
subflowResultCacheId := fmt.Sprintf("%s_%s_subflowresult", res.ExecutionId, result.Action.ID)
cache, err := GetCache(ctx, subflowResultCacheId)
if err == nil {
cacheData := []byte(cache.([]uint8))
//log.Printf("[DEBUG] Cachedata for other subflow: '%s'", string(cacheData))
if len(cacheData) > 0 {
res.Result = string(cacheData)
res.ResultSet = true
finishedSubflows += 1
} else {
DeleteCache(ctx, subflowResultCacheId)
}
} else {
// Get the workflow execution for the subflow
// Can't do this, as it causes an infinite loop?
// This function is used in GetWorkflowExecution
subflowExecution, err := GetWorkflowExecution(ctx, res.ExecutionId)
//log.Printf("[DEBUG][%s] Got subflow execution: %s", res.ExecutionId, subflowExecution.Status)
if err != nil {
log.Printf("[ERROR] Failed getting subflow execution: %s", subflowExecution.Status)
} else {
if subflowExecution.Status == "EXECUTING" {
//DeleteCache(ctx, fmt.Sprintf("workflowexecution_%s", res.ExecutionId))
} else if subflowExecution.Status != "EXECUTING" {
// Ensure it gets the last result based on CompletedAt
//log.Printf("[DEBUG] NOT EXECUTING!!")
foundResult := ActionResult{}
for _, result := range subflowExecution.Results {
if result.Status == "SUCCESS" && result.CompletedAt >= foundResult.CompletedAt {
foundResult = result
}
}
if len(foundResult.Result) > 0 {
res.Result = foundResult.Result
}
if len(res.Result) == 0 || subflowExecution.Status == "ABORTED" {
// Find the last result and use that
res.Result = subflowExecution.Workflow.DefaultReturnValue
}
if len(subflowExecution.Result) > 0 {
res.Result = subflowExecution.Result
}
res.ResultSet = true
finishedSubflows += 1
if len(res.Result) > 0 {
SetCache(ctx, subflowResultCacheId, []byte(subflowExecution.Result), 60)
}
}
}
}
newResults = append(newResults, res)
}
baseResultData, err := json.Marshal(newResults)
if err != nil {
log.Printf("[ERROR] Failed marshalling subflow loop request data (1): %s", err)
return result, false
}
result.Result = string(baseResultData)
if finishedSubflows == len(newResults) {
//log.Printf("[DEBUG] Finished sub result from caching?")
// Status is used to determine if the current subflow is finished
if failedCount == finishedSubflows {
result.Status = "FAILURE"
} else {
result.Status = "SUCCESS"
}
if result.CompletedAt == 0 {
result.CompletedAt = time.Now().Unix() * 1000
}
} else {
//log.Printf("[DEBUG] Not finished sub result from caching yet")
}
return result, true
}
func ValidateRequestOverload(resp http.ResponseWriter, request *http.Request, amount ...int) error {
// 1. Get current amount of requests for the user
// 2. Check if the user is allowed to make more requests
// 3. If not, return error
// 4. If yes, continue and add the request to the list
// Use the GetCache() and SetCache() functions to store the request count
maxAmount := 4
if len(amount) > 0 {
maxAmount = amount[0]
}
// Max amount per minute
foundIP := GetRequestIp(request)
portRemoval := strings.Split(foundIP, ":")
if len(portRemoval) > 1 {
foundIP = strings.Join(portRemoval[:len(portRemoval)-1], ":")
}
//log.Printf("\n\n\nIP: %s\n\n\n", foundIP)
if foundIP == "" || foundIP == "127.0.0.1" || foundIP == "::1" || foundIP == "[::1]" {
if debug {
log.Printf("[DEBUG] Skipping request overload check for IP: %s", foundIP)
}
return nil
}
// Check if the foundIP includes ONE colon for the port
if strings.Count(foundIP, ":") == 1 {
foundIP = strings.Split(foundIP, ":")[0]
}
timenow := time.Now().Unix()
userRequest := UserRequest{
IP: foundIP,
Method: request.Method,
Path: request.URL.Path,
Timestamp: timenow,
}
requestList := []UserRequest{}
// Maybe do per path? Idk
ctx := GetContext(request)
cacheKey := fmt.Sprintf("userrequest_%s", userRequest.IP)
cache, err := GetCache(ctx, cacheKey)
if err != nil {
//log.Printf("[ERROR] Failed getting cache for key %s: %s", cacheKey, err)
requestList = append(requestList, userRequest)
b, err := json.Marshal(requestList)
if err != nil {
log.Printf("[WARNING] Failed marshalling requestlist: %s", err)
return nil
}
// Set cache for 1 minute
err = SetCache(ctx, cacheKey, b, 1)
if err != nil {
log.Printf("[ERROR] Failed setting cache for key %s: %s", cacheKey, err)
return nil
}
return nil
}
// Parse out the data in the cache
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &requestList)
if err != nil {
log.Printf("[WARNING] Failed unmarshalling requestlist: %s", err)
return nil
}
// Remove any item more than 60 seconds back to make a sliding window
newList := []UserRequest{}
for _, req := range requestList {
if req.Timestamp < (timenow - 60) {
continue
}
newList = append(newList, req)
}
if len(newList) >= maxAmount {
// FIXME: Should we add to the list even if we return an error?
return errors.New("Too many requests")
}
//log.Printf("[DEBUG] Adding request to list")
newList = append(newList, userRequest)
b, err := json.Marshal(newList)
if err != nil {
log.Printf("[ERROR] Failed marshalling requestlist: %s", err)
return nil
}
// Set cache for 1 minute
err = SetCache(ctx, cacheKey, b, 1)
if err != nil {
log.Printf("[ERROR] Failed setting cache for key %s: %s", cacheKey, err)
}
return nil
}
func DistributeAppToEnvironments(ctx context.Context, org Org, appnames []string) error {
envs, err := GetEnvironments(ctx, org.Id)
if err != nil {
log.Printf("[ERROR] Failed getting environments for org: %s", err)
return err
}
for appIndex, appname := range appnames {
appnames[appIndex] = strings.ReplaceAll(appname, " ", "-")
}
if len(envs) > 10 {
envs = envs[:10]
}
// Should add to queues in the current org
for _, env := range envs {
if env.Archived {
continue
}
if strings.ToLower(env.Name) == "cloud" {
continue
}
log.Printf("[DEBUG] Distributing app image '%s' to environment: %s", strings.Join(appnames, ", "), env.Name)
// Add to the queue
request := ExecutionRequest{
Type: "DOCKER_IMAGE_DOWNLOAD",
ExecutionId: uuid.NewV4().String(),
ExecutionArgument: fmt.Sprintf("%s,%s", strings.ToLower(strings.Join(appnames, ",")), strings.Join(appnames, ",")),
Priority: 11,
}
parsedId := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(env.Name, " ", "-"), "_", "-")), org.Id)
if project.Environment != "cloud" {
parsedId = strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(env.Name, " ", "-"), "_", "-"))
}
err = SetWorkflowQueue(ctx, request, parsedId)
if err != nil {
log.Printf("[ERROR] Failed setting workflow queue for env: %s", err)
continue
}
}
return nil
}
func fixOrgUsers(ctx context.Context, foundOrg Org) error {
if project.Environment == "cloud" {
log.Printf("[DEBUG] Skipping fixOrgUsers for cloud")
return errors.New("Not updating cloud")
}
if len(foundOrg.Users) != 0 {
return errors.New("Org already has users")
}
users, countErr := GetAllUsers(ctx)
if countErr != nil {
log.Printf("[ERROR] Failed getting all users in auto fix org users: %s", countErr)
return countErr
}
log.Printf("[DEBUG] Found %d users to potentially add to org %s", len(users), foundOrg.Id)
for _, user := range users {
if !ArrayContains(user.Orgs, foundOrg.Id) {
continue
}
log.Printf("[DEBUG] Re-adding user %s (%s) to org %s (%s)", user.Username, user.Id, foundOrg.Name, foundOrg.Id)
user.Role = "admin"
foundOrg.Users = append(foundOrg.Users, user)
}
// Save the org
err := SetOrg(ctx, foundOrg, foundOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed saving org %s while readding a user: %s", foundOrg.Id, err)
return err
}
return nil
}
func HandleCheckLicense(ctx context.Context, org Org) Org {
// ==== LICENSE BYPASS PATCH ====
org.Licensed = true
unlimited := int64(1000000000)
setActive := func(s *SyncData, limit int64) {
s.Active = true
s.Limit = limit
}
setActive(&org.SyncFeatures.AppExecutions, unlimited)
setActive(&org.SyncFeatures.OnpremAppExecutions, unlimited)
setActive(&org.SyncFeatures.MultiEnv, unlimited)
setActive(&org.SyncFeatures.MultiTenant, unlimited)
setActive(&org.SyncFeatures.MultiRegion, unlimited)
setActive(&org.SyncFeatures.Webhook, unlimited)
setActive(&org.SyncFeatures.Schedules, unlimited)
setActive(&org.SyncFeatures.UserInput, unlimited)
setActive(&org.SyncFeatures.SendMail, unlimited)
setActive(&org.SyncFeatures.SendSms, unlimited)
setActive(&org.SyncFeatures.Updates, unlimited)
setActive(&org.SyncFeatures.EmailTrigger, unlimited)
setActive(&org.SyncFeatures.Notifications, unlimited)
setActive(&org.SyncFeatures.Workflows, unlimited)
setActive(&org.SyncFeatures.Autocomplete, unlimited)
setActive(&org.SyncFeatures.WorkflowExecutions, unlimited)
setActive(&org.SyncFeatures.Authentication, unlimited)
setActive(&org.SyncFeatures.Schedule, unlimited)
setActive(&org.SyncFeatures.Apps, unlimited)
setActive(&org.SyncFeatures.ShuffleGPT, unlimited)
setActive(&org.SyncFeatures.Branding, unlimited)
setActive(&org.SyncFeatures.AgentExecutions, unlimited)
setActive(&org.SyncFeatures.AgentTokens, unlimited)
return org
// ===== END LICENSE BYPASS PATCH =====
if project.Environment == "cloud" {
return org
}
shuffleLicenseKey := os.Getenv("SHUFFLE_LICENSE")
if org.CloudSync {
cacheKey := fmt.Sprintf("org_sync_features_%s", org.Id)
syncFeatures, err := GetCache(ctx, cacheKey)
if err != nil {
//log.Printf("[ERROR] Failed to get cache in HandleCheckLicense: %v", err)
org.SyncFeatures.MultiEnv.Active = false
org.SyncFeatures.MultiEnv.Limit = 1
org.SyncFeatures.MultiTenant.Active = false
org.SyncFeatures.MultiTenant.Limit = 3
org.SyncFeatures.Branding.Active = false
org.Licensed = false
org.SyncFeatures.AppExecutions.Active = false
org.SyncFeatures.AppExecutions.Limit = 25000
return org
}
features := SyncFeatures{}
if data, ok := syncFeatures.([]byte); ok {
if err := json.Unmarshal(data, &features); err == nil {
licenseCacheKey := fmt.Sprintf("org_licensed_%s", org.Id)
licensed, err := GetCache(ctx, licenseCacheKey)
if err != nil {
org.Licensed = false
} else if data, ok := licensed.([]byte); ok {
licenseData := string(data)
if licenseData == "true" {
org.Licensed = true
org.SyncFeatures.MultiEnv.Active = features.MultiEnv.Active
org.SyncFeatures.MultiEnv.Limit = features.MultiEnv.Limit
org.SyncFeatures.MultiTenant.Active = features.MultiTenant.Active
org.SyncFeatures.MultiTenant.Limit = features.MultiTenant.Limit
org.SyncFeatures.Branding.Active = features.Branding.Active
org.SyncFeatures.AppExecutions.Active = features.OnpremAppExecutions.Active
if features.OnpremAppExecutions.Limit < 25000 {
org.SyncFeatures.AppExecutions.Limit = 25000
} else {
org.SyncFeatures.AppExecutions.Limit = features.OnpremAppExecutions.Limit
}
} else {
org.SyncFeatures.MultiEnv.Active = false
org.SyncFeatures.MultiEnv.Limit = 1
org.SyncFeatures.MultiTenant.Active = false
org.SyncFeatures.MultiTenant.Limit = 3
org.SyncFeatures.Branding.Active = false
org.SyncFeatures.AppExecutions.Active = false
org.SyncFeatures.AppExecutions.Limit = 25000
}
} else {
org.Licensed = false
org.SyncFeatures.MultiEnv.Active = false
org.SyncFeatures.MultiEnv.Limit = 1
org.SyncFeatures.MultiTenant.Active = false
org.SyncFeatures.MultiTenant.Limit = 3
org.SyncFeatures.Branding.Active = false
org.SyncFeatures.AppExecutions.Active = false
org.SyncFeatures.AppExecutions.Limit = 25000
}
org.SyncFeatures.AppExecutions.Active = features.OnpremAppExecutions.Active
if features.OnpremAppExecutions.Limit < 25000 {
org.SyncFeatures.AppExecutions.Limit = 25000
} else {
org.SyncFeatures.AppExecutions.Limit = features.OnpremAppExecutions.Limit
}
org.SyncFeatures.Webhook.Active = features.Webhook.Active
org.SyncFeatures.Webhook.Limit = features.Webhook.Limit
org.SyncFeatures.Schedules.Active = features.Schedules.Active
org.SyncFeatures.Schedules.Limit = features.Schedules.Limit
org.SyncFeatures.UserInput.Active = features.UserInput.Active
org.SyncFeatures.UserInput.Limit = features.UserInput.Limit
org.SyncFeatures.SendMail.Active = features.SendMail.Active
org.SyncFeatures.SendMail.Limit = features.SendMail.Limit
org.SyncFeatures.SendSms.Active = features.SendSms.Active
org.SyncFeatures.SendSms.Limit = features.SendSms.Limit
org.SyncFeatures.Updates.Active = features.Updates.Active
org.SyncFeatures.Updates.Limit = features.Updates.Limit
org.SyncFeatures.EmailTrigger.Active = features.EmailTrigger.Active
org.SyncFeatures.EmailTrigger.Limit = features.EmailTrigger.Limit
org.SyncFeatures.Notifications.Active = features.Notifications.Active
org.SyncFeatures.Notifications.Limit = features.Notifications.Limit
org.SyncFeatures.Workflows.Active = features.Workflows.Active
org.SyncFeatures.Workflows.Limit = features.Workflows.Limit
org.SyncFeatures.Autocomplete.Active = features.Autocomplete.Active
org.SyncFeatures.Autocomplete.Limit = features.Autocomplete.Limit
org.SyncFeatures.Authentication.Active = features.Authentication.Active
org.SyncFeatures.Authentication.Limit = features.Authentication.Limit
org.SyncFeatures.WorkflowExecutions.Active = features.WorkflowExecutions.Active
org.SyncFeatures.WorkflowExecutions.Limit = features.WorkflowExecutions.Limit
org.SyncFeatures.Schedule.Active = features.Schedule.Active
org.SyncFeatures.Schedule.Limit = features.Schedule.Limit
org.SyncFeatures.Apps.Active = features.Apps.Active
org.SyncFeatures.Apps.Limit = features.Apps.Limit
org.SyncFeatures.ShuffleGPT.Active = features.ShuffleGPT.Active
org.SyncFeatures.ShuffleGPT.Limit = features.ShuffleGPT.Limit
if !features.MultiTenant.Active && features.MultiTenant.Limit > 3 {
org.SyncFeatures.MultiTenant.Limit = 3
}
}
} else {
org.Licensed = false
org.SyncFeatures.MultiEnv.Active = false
org.SyncFeatures.MultiEnv.Limit = 1
org.SyncFeatures.MultiTenant.Active = false
org.SyncFeatures.MultiTenant.Limit = 3
org.SyncFeatures.Branding.Active = false
org.SyncFeatures.AppExecutions.Active = false
org.SyncFeatures.AppExecutions.Limit = 25000
}
if len(shuffleLicenseKey) > 0 {
license := checkNoInternet()
if license.Valid == true {
org.Licensed = true
if license.Environment.Limit > org.SyncFeatures.MultiEnv.Limit {
org.SyncFeatures.MultiEnv.Limit = license.Environment.Limit
org.SyncFeatures.MultiEnv.Active = license.Environment.Active
}
if license.Tenant.Limit > org.SyncFeatures.MultiTenant.Limit {
org.SyncFeatures.MultiTenant.Limit = license.Tenant.Limit
org.SyncFeatures.MultiTenant.Active = license.Tenant.Active
}
if license.AppRuns.Limit > org.SyncFeatures.AppExecutions.Limit {
org.SyncFeatures.AppExecutions.Limit = license.AppRuns.Limit
org.SyncFeatures.AppExecutions.Active = license.AppRuns.Active
}
org.SyncFeatures.Branding.Active = license.Branding
}
}
subscriptionCacheKey := fmt.Sprintf("org_subscriptions_%s", org.Id)
cachedData, err := GetCache(ctx, subscriptionCacheKey)
if err != nil {
log.Printf("[ERROR] Failed to get cache for org (%s) subscriptions in HandleCheckLicense: %v", org.Id, err)
return org
} else if data, ok := cachedData.([]byte); ok {
subscriptionsList := []PaymentSubscription{}
if err := json.Unmarshal(data, &subscriptionsList); err == nil {
org.Subscriptions = subscriptionsList
} else {
log.Printf("[ERROR] Failed to parse cached subscriptions for org (%s) in HandleCheckLicense: %v", org.Id, err)
}
}
} else if len(shuffleLicenseKey) > 0 {
license := checkNoInternet()
if license.Valid == true {
org.Licensed = true
org.SyncFeatures.MultiEnv.Limit = license.Environment.Limit
org.SyncFeatures.MultiEnv.Active = license.Environment.Active
org.SyncFeatures.MultiTenant.Limit = license.Tenant.Limit
org.SyncFeatures.MultiTenant.Active = license.Tenant.Active
org.SyncFeatures.Branding.Active = license.Branding
org.SyncFeatures.AppExecutions.Active = license.AppRuns.Active
org.SyncFeatures.AppExecutions.Limit = license.AppRuns.Limit
org.SyncFeatures.WorkflowExecutions.Active = true
org.SyncFeatures.Webhook.Active = true
org.SyncFeatures.Schedules.Active = true
org.SyncFeatures.UserInput.Active = true
org.SyncFeatures.SendMail.Active = true
org.SyncFeatures.SendSms.Active = true
org.SyncFeatures.Updates.Active = true
org.SyncFeatures.EmailTrigger.Active = true
org.SyncFeatures.Notifications.Active = true
org.SyncFeatures.Workflows.Active = true
org.SyncFeatures.Autocomplete.Active = true
org.SyncFeatures.Authentication.Active = true
org.SyncFeatures.Schedule.Active = true
org.SyncFeatures.Apps.Active = true
org.SyncFeatures.ShuffleGPT.Active = true
} else {
org.Licensed = false
org.SyncFeatures.MultiEnv.Limit = 1
org.SyncFeatures.MultiEnv.Active = false
org.SyncFeatures.MultiTenant.Limit = 3
org.SyncFeatures.MultiTenant.Active = false
org.SyncFeatures.Branding.Active = false
org.SyncFeatures.AppExecutions.Active = false
org.SyncFeatures.AppExecutions.Limit = 25000
}
parsedEula := GetOnpremPaidEula()
if debug {
log.Printf("[DEBUG] Org has the Enterprise License Key")
}
var endDate int64
var cancellationDate int64
active := false
features := []string{
"∞ Days Workflow Run History",
"∞ Days Workflow Backup",
"∞ Users",
"Air Gap Environment",
"Critical Response",
"On-Call Support",
"Setup and Maintenance",
"Key Management System",
"Custom Integrations",
"Custom Scaling Options",
"Billing and Invoice Included",
"Custom Contract",
}
if license.Valid {
parsedTimeout, err := time.Parse("02-01-2006", license.Timeout)
if err != nil {
log.Printf("[ERROR] Failed parsing license timeout: %s", err)
parsedTimeout = time.Now()
}
endDate = parsedTimeout.Unix()
cancellationDate = 0
active = true
} else {
endDate = time.Now().Unix()
cancellationDate = time.Now().Unix()
active = false
}
subscription := PaymentSubscription{
Name: "Enterprise License",
Active: active,
CancellationDate: cancellationDate,
SupportLevel: "Enterprise Support",
Startdate: time.Now().Unix(),
Enddate: endDate,
Recurrence: string("monthly"),
Amount: "0",
Currency: string("USD"),
Level: "1",
Reference: "",
Limit: 1,
Features: features,
EulaSigned: true,
Eula: parsedEula,
}
org.Subscriptions = []PaymentSubscription{subscription}
} else {
if debug {
log.Printf("[DEBUG] Org %v does not have an enterprise license. Please purchase an enterprise license to unlock production-ready features. Contact support@shuffler.io for more information.", org.Id)
}
org.Licensed = false
org.SyncFeatures.MultiEnv.Limit = 1
org.SyncFeatures.MultiEnv.Active = false
org.SyncFeatures.MultiTenant.Limit = 3
org.SyncFeatures.MultiTenant.Active = false
org.SyncFeatures.Branding.Active = false
org.SyncFeatures.AppExecutions.Active = false
org.SyncFeatures.AppExecutions.Limit = 25000
}
return org
}
func IsLicensed(ctx context.Context, org Org) bool {
if project.Environment == "cloud" && len(org.ManagerOrgs) > 0 {
return true
}
if len(org.SubscriptionUserId) == 0 {
return false
}
//if len(org.Subscriptions) > 0 {
// return true
//}
environments, err := GetEnvironments(ctx, org.Id)
if err != nil {
log.Printf("[ERROR] Failed getting environments for org %s: %s", org.Id, err)
return false
}
for _, env := range environments {
if env.Archived {
continue
}
//if env.Licensed {
// return true
//}
}
return false
}
// Generates a standard destination workflow that uses:
// 1. A Startnode mapping $exec
// 2. An enrichment subflow that maps the data from $exec
// - A data merger of 1 & 2
// - Integration framework with dest app
func GetStandardDestWorkflow(app *WorkflowApp, action string, enrich bool) *Workflow {
appname := app.Name
appCategory := ""
if len(app.Categories) > 0 {
appCategory = app.Categories[0]
}
workflowId := uuid.NewV4().String()
startnodeId := uuid.NewV4().String()
workflow := Workflow{
ID: workflowId,
Start: startnodeId,
}
workflow.Actions = append(workflow.Actions, Action{
AppName: "Shuffle Tools",
AppVersion: "1.2.0",
Label: "create_startnode",
ID: startnodeId,
Name: "repeat_back_to_me",
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "call",
Value: "$exec",
Multiline: true,
},
},
Position: Position{
X: 0,
Y: 0,
},
})
previousnodeId := startnodeId
previousnodeRef := fmt.Sprintf("$%s", workflow.Actions[0].Label)
if enrich {
enrichNodeId := uuid.NewV4().String()
workflow.Triggers = append(workflow.Triggers, Trigger{
AppName: "Shuffle Workflow",
AppVersion: "1.0.0",
Name: "Shuffle Workflow",
ID: enrichNodeId,
Label: "Enrich",
Tags: []string{"Enrich"},
TriggerType: "SUBFLOW",
Position: Position{
X: 150,
Y: 150,
},
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "workflow",
Value: "",
},
WorkflowAppActionParameter{
Name: "argument",
Value: "$exec",
},
WorkflowAppActionParameter{
Name: "user_apikey",
Value: "",
},
WorkflowAppActionParameter{
Name: "startnode",
Value: "",
},
WorkflowAppActionParameter{
Name: "check_result",
Value: "true",
},
},
})
// Start -> subflow node
workflow.Branches = append(workflow.Branches, Branch{
ID: uuid.NewV4().String(),
SourceID: startnodeId,
DestinationID: enrichNodeId,
})
// Add merge node
mergeNodeId := uuid.NewV4().String()
workflow.Actions = append(workflow.Actions, Action{
AppName: "Shuffle Tools",
AppVersion: "1.2.0",
Label: "merge enrichment",
ID: mergeNodeId,
Name: "merge_incoming_branches",
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "input_type",
Value: "dict",
Options: []string{"list", "dict"},
Required: true,
},
},
Position: Position{
X: 0,
Y: 300,
},
})
// Start -> subflow node
workflow.Branches = append(workflow.Branches, Branch{
ID: uuid.NewV4().String(),
SourceID: startnodeId,
DestinationID: mergeNodeId,
})
workflow.Branches = append(workflow.Branches, Branch{
ID: uuid.NewV4().String(),
SourceID: enrichNodeId,
DestinationID: mergeNodeId,
})
previousnodeId = mergeNodeId
previousnodeRef = fmt.Sprintf("$%s", strings.ReplaceAll(workflow.Actions[1].Label, " ", "_"))
}
integrationFrameworkId := uuid.NewV4().String()
workflow.Actions = append(workflow.Actions, Action{
AppName: "Integration Framework",
AppVersion: "1.0.0",
AppID: "integration",
Label: strings.ReplaceAll(action, " ", "_"),
ID: integrationFrameworkId,
Name: appCategory,
LargeImage: app.LargeImage,
Parameters: []WorkflowAppActionParameter{
WorkflowAppActionParameter{
Name: "action",
Value: action,
Options: []string{action},
Required: true,
},
WorkflowAppActionParameter{
Name: "fields",
Value: previousnodeRef,
Multiline: true,
},
WorkflowAppActionParameter{
Name: "app_name",
Value: appname,
},
},
Position: Position{
X: 0,
Y: 450,
},
})
workflow.Branches = append(workflow.Branches, Branch{
ID: uuid.NewV4().String(),
SourceID: previousnodeId,
DestinationID: integrationFrameworkId,
})
return &workflow
}
func CheckSessionOrgs(ctx context.Context, user User) {
if !ArrayContains(user.ValidatedSessionOrgs, user.ActiveOrg.Id) {
user.ValidatedSessionOrgs = append(user.ValidatedSessionOrgs, user.ActiveOrg.Id)
err := SetUser(ctx, &user, false)
if err != nil {
log.Printf("[ERROR] Failed setting validated session orgs for user %s: %s", user.Username, err)
}
}
}
// Handles statistics incrementation for workflow executions
func HandleExecutionCacheIncrement(ctx context.Context, execution WorkflowExecution) {
if execution.Status != "FINISHED" && execution.Status != "ABORTED" && execution.Status != "FAILURE" {
//log.Printf("[DEBUG] Execution %s is not finished (%s). Not incrementing cache", execution.ExecutionId, execution.Status)
return
}
cacheIncrementKey := fmt.Sprintf("%s_cacheset", execution.ExecutionId)
_, err := GetCache(ctx, cacheIncrementKey)
if err == nil {
//log.Printf("[DEBUG] Cache already incremented for execution %s", execution.ExecutionId)
return
}
SetCache(ctx, cacheIncrementKey, []byte{1}, 60)
env := ""
appruns := 0
appfailure := 0
subflows := 0
for _, result := range execution.Results {
// Shoud all be the same :)
if len(result.Action.Environment) > 0 {
env = result.Action.Environment
}
if result.Status == "SUCCESS" {
appruns += 1
} else if result.Status == "FAILURE" || result.Status == "ABORTED" {
appruns += 1
appfailure += 1
}
if result.Action.AppName == "Shuffle Workflow" && result.Status == "SUCCESS" {
subflows += 1
}
}
actionLabelSuccess := map[string]int{}
actionLabelFails := map[string]int{}
for _, action := range execution.Workflow.Actions {
if len(action.Environment) > 0 {
env = action.Environment
}
if len(action.CategoryLabel) == 0 {
continue
}
categoryLabel := strings.ToLower(strings.ReplaceAll(action.CategoryLabel[0], " ", "_"))
for _, result := range execution.Results {
if result.Action.ID != action.ID {
continue
}
if result.Status == "SUCCESS" {
// Check the result if result.Result.status < 300 or something similar
updateValue := true
outputValue := HTTPOutput{}
err := json.Unmarshal([]byte(result.Result), &outputValue)
if err == nil {
if !outputValue.Success || outputValue.Status >= 300 {
result.Status = "ABORTED"
updateValue = false
}
}
if updateValue {
if _, ok := actionLabelSuccess[categoryLabel]; ok {
actionLabelSuccess[categoryLabel] += 1
} else {
actionLabelSuccess[categoryLabel] = 1
}
}
}
if result.Status == "FAILURE" || result.Status == "ABORTED" {
if _, ok := actionLabelFails[categoryLabel]; ok {
actionLabelFails[categoryLabel] += 1
} else {
actionLabelFails[categoryLabel] = 1
}
} else {
// Skipped or something. Not relevant.
}
}
}
if appruns > 0 {
apprunName := fmt.Sprintf("app_executions_%s", env)
if len(env) == 0 {
apprunName = fmt.Sprintf("app_executions")
}
IncrementCache(ctx, execution.ExecutionOrg, apprunName, appruns)
}
if appfailure > 0 {
IncrementCache(ctx, execution.ExecutionOrg, "app_executions_failed", appfailure)
}
if subflows > 0 {
IncrementCache(ctx, execution.ExecutionOrg, "subflow_executions", subflows)
}
if execution.Status == "ABORTED" {
IncrementCache(ctx, execution.ExecutionOrg, "workflow_executions_failed")
} else if execution.Status == "FINISHED" {
IncrementCache(ctx, execution.ExecutionOrg, "workflow_executions_finished")
} else {
IncrementCache(ctx, execution.ExecutionOrg, "workflow_executions_executing")
}
for key, value := range actionLabelSuccess {
IncrementCache(ctx, execution.ExecutionOrg, fmt.Sprintf("singul_success_%s", key), value)
}
for key, value := range actionLabelFails {
IncrementCache(ctx, execution.ExecutionOrg, fmt.Sprintf("singul_fail_%s", key), value)
}
}
func GetChildWorkflows(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Removed check here as it may be a public workflow
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in getting child workflows: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`))
return
}
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Workflow %s doesn't exist.", fileId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed finding workflow"}`))
return
}
if len(workflow.ID) == 0 || len(workflow.Name) == 0 {
log.Printf("[WARNING] Workflow %s is not valid. Missing ID or Name.", fileId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Workflow is not valid"}`))
return
}
// FIXME: Check if this workflow has a parent workflow
if len(workflow.ParentWorkflowId) > 0 && workflow.ParentWorkflowId != fileId {
workflow, err = GetWorkflow(ctx, workflow.ParentWorkflowId)
if err != nil {
log.Printf("[WARNING] Parent workflow %s doesn't exist.", workflow.ParentWorkflowId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Failed finding parent workflow"}`))
return
}
// Updating role
orgUserFound := false
for _, orgId := range user.Orgs {
if orgId != workflow.OrgId {
continue
}
org, err := GetOrg(ctx, orgId)
if err != nil {
log.Printf("[WARNING] Failed getting org during parent org loading %s: %s", org.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false}`))
return
}
for _, orgUser := range org.Users {
if user.Id == orgUser.Id {
user.Role = orgUser.Role
user.ActiveOrg.Id = org.Id
orgUserFound = true
}
}
break
}
if !orgUserFound {
log.Printf("[WARNING] User %s not found in parent org %s", user.Username, workflow.OrgId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "User not found in parent org"}`))
return
}
}
// Check workflow.Sharing == private / public / org too
if user.Id != workflow.Owner || len(user.Id) == 0 {
// Added org-reader as the user should be able to read everything in an org
//if workflow.OrgId == user.ActiveOrg.Id && (user.Role == "admin" || user.Role == "org-reader") {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (get child workflows)", user.Username, workflow.ID)
// Only for Read-Only. No executions or impersonations.
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access child workflows for %s", user.Username, workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (get child workflow). Verified: %t, Active: %t, SupportAccess: %t, Username: %s", user.Username, workflow.ID, user.Verified, user.Active, user.SupportAccess, user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// Access is granted -> get revisions
childWorkflows, err := ListChildWorkflows(ctx, workflow.ID)
if err != nil {
log.Printf("[WARNING] Failed getting child workflows of %s: %s", workflow.ID, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
newWfs := []Workflow{}
for _, wf := range childWorkflows {
if wf.ParentWorkflowId != workflow.ID {
continue
}
newWfs = append(newWfs, wf)
}
body, err := json.Marshal(newWfs)
if err != nil {
log.Printf("[WARNING] Failed child workflow GET marshalling: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(body)
}
// Checks & validates workflow based on last few runs~
func GetWorkflowValidation(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Removed check here as it may be a public workflow
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in getting specific workflow: %s. Continuing because it may be public.", err)
}
location := strings.Split(request.URL.String(), "/")
var fileId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fileId = location[4]
}
if strings.Contains(fileId, "?") {
fileId = strings.Split(fileId, "?")[0]
}
if len(fileId) != 36 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Workflow ID when getting workflow is not valid"}`))
return
}
ctx := GetContext(request)
workflow, err := GetWorkflow(ctx, fileId)
if err != nil {
log.Printf("[WARNING] Workflow %s doesn't exist.", fileId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed finding workflow"}`))
return
}
// Check workflow.Sharing == private / public / org too
if user.Id != workflow.Owner || len(user.Id) == 0 {
// Added org-reader as the user should be able to read everything in an org
//if workflow.OrgId == user.ActiveOrg.Id && (user.Role == "admin" || user.Role == "org-reader") {
if workflow.OrgId == user.ActiveOrg.Id {
log.Printf("[AUDIT] User %s is accessing workflow %s as admin (get workflow revisions)", user.Username, workflow.ID)
// Only for Read-Only. No executions or impersonations.
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] Letting verified support admin %s access workflow revisions for %s", user.Username, workflow.ID)
} else {
log.Printf("[AUDIT] Wrong user (%s) for workflow %s (get workflow revisions). Verified: %t, Active: %t, SupportAccess: %t, Username: %s", user.Username, workflow.ID, user.Verified, user.Active, user.SupportAccess, user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
}
// FIXME: Check last 10 executions + notifications if they
// Make sure it adds subflows as well and highlights failing apps
// Access is granted -> get revisions
resp.Write([]byte(`{"success": false, "reason": "Not implemented"}`))
resp.WriteHeader(500)
}
func HandleUserPrivateTraining(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
err := ValidateRequestOverload(resp, request)
if err != nil {
log.Printf("[INFO] Request overload for IP %s in private training", GetRequestIp(request))
resp.WriteHeader(http.StatusTooManyRequests)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Too many requests"}`)))
return
}
gceProject := os.Getenv("SHUFFLE_GCEPROJECT")
if gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
log.Printf("[DEBUG] Redirecting training request to main site handler (shuffler.io). Project: %s", gceProject)
RedirectUserRequest(resp, request)
return
}
User, userErr := HandleApiAuthentication(resp, request)
if userErr != nil {
log.Printf("[AUDIT] Api authentication failed in private training: %s", userErr)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
type TrainingData struct {
OrgId string `json:"org_id" datastore:"org_id"`
Training string `json:"trainingMembers" datastore:"trainingMembers"`
Message string `json:"message" datastore:"message"`
}
var tmpData TrainingData
err = json.Unmarshal(body, &tmpData)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling test: %s", err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false}`))
return
}
if len(tmpData.OrgId) == 0 || len(tmpData.Training) == 0 {
log.Printf("[WARNING] Missing org_id or training in private training request")
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false, "reason": "Missing org_id or training"}`))
return
}
//Get user org
ctx := GetContext(request)
org, err := GetOrg(ctx, tmpData.OrgId)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", tmpData.OrgId, err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false}`))
return
}
email := []string{User.Username}
Subject := "Thank you for your private training request"
Message := fmt.Sprintf("Hi there, Thank you for submitting request for shuffle private training. This is confirmation that we have received your private training request. You have requested a private training for %v members. We will get back to you shortly.
Best Regards
Shuffle Team", tmpData.Training)
err = sendMailSendgrid(email, Subject, Message, false, []string{})
if err != nil {
log.Printf("[ERROR] Failed sending mail: %s", err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false}`))
return
}
//Send mail to the shuffle support
email = []string{"support@shuffler.io"}
Subject = fmt.Sprintf("Private training request")
Message = fmt.Sprintf("Private training request :
Org id: %v
Org Name: %v
Username: %v
Training Members: %v
Customer: %v
Message: %v", org.Id, org.Name, User.Username, tmpData.Training, org.LeadInfo.Customer, tmpData.Message)
err = sendMailSendgrid(email, Subject, Message, false, []string{})
if err != nil {
log.Printf("[ERROR] Failed sending mail: %s", err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(`{"success": false}`))
return
}
log.Printf("[INFO] Private training request from %s for %s members. Message: %s", org.Org, tmpData.Training, tmpData.Message)
resp.WriteHeader(http.StatusOK)
resp.Write([]byte(`{"success": true}`))
}
// An API to ONLY return PUBLIC forms for an org
// A public form = Workflow with "sharing": form
func HandleGetOrgForms(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
err := ValidateRequestOverload(resp, request)
if err != nil {
log.Printf("[INFO] Request overload for IP %s Get Org Forms", GetRequestIp(request))
resp.WriteHeader(429)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Too many requests"}`)))
return
}
var orgId string
location := strings.Split(request.URL.String(), "/")
if location[1] == "api" {
if len(location) <= 4 {
log.Printf("Path too short: %d", len(location))
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
orgId = location[4]
}
if strings.Contains(orgId, "?") {
orgId = strings.Split(orgId, "?")[0]
}
validAuth := false
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[AUDIT] Api authentication failed in getting forms: %s. Allowing anyway", err)
} else {
if len(user.Id) > 0 && len(user.Username) > 0 {
if user.ActiveOrg.Id == orgId {
validAuth = true
}
}
}
if len(orgId) < 36 || len(orgId) > 36 {
log.Printf("[WARNING] Bad ID '%s' of length %d when getting forms is not valid", orgId, len(orgId))
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Org ID when getting forms is not valid"}`))
return
}
// Load the org to see if it wants them public or not
ctx := GetContext(request)
org, err := GetOrg(ctx, orgId)
if err != nil {
log.Printf("[WARNING] Org %s doesn't exist.", orgId)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Failed finding org"}`))
return
}
log.Printf("[INFO] Getting forms for org %s (%s)", org.Name, org.Id)
// Prevent cache steals in any way
randomUserId := uuid.NewV4().String()
randomUser := User{
Id: randomUserId,
ActiveOrg: OrgMini{
Id: orgId,
Name: org.Name,
},
}
if validAuth {
randomUser = user
}
workflows, err := GetAllWorkflowsByQuery(ctx, randomUser, 50, "")
if err != nil {
log.Printf("[WARNING] Failed getting workflows for user %s (0): %s", randomUser.Username, err)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
if len(workflows) == 0 {
log.Printf("[INFO] No workflows found for user %s (%s) in org %s (%s)", randomUser.Username, randomUser.Id, randomUser.ActiveOrg.Name, randomUser.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte("[]"))
return
}
relevantForms := []Workflow{}
for _, workflow := range workflows {
if validAuth {
if len(workflow.InputQuestions) == 0 && len(workflow.FormControl.InputMarkdown) == 0 {
continue
}
if workflow.Sharing == "form" {
relevantForms = append(relevantForms, workflow)
continue
}
} else {
if workflow.Sharing != "form" {
continue
}
// Overwrite to remove anything unecessary for most locations
workflow = Workflow{
Name: workflow.Name,
ID: workflow.ID,
Owner: workflow.Owner,
OrgId: workflow.OrgId,
FormControl: workflow.FormControl,
Sharing: workflow.Sharing,
Description: workflow.Description,
InputQuestions: workflow.InputQuestions,
}
}
relevantForms = append(relevantForms, workflow)
}
if len(relevantForms) == 0 {
log.Printf("[INFO] No forms found for user '%s' (%s) in org %s (%s)", randomUser.Username, randomUser.Id, randomUser.ActiveOrg.Name, randomUser.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte("[]"))
return
}
log.Printf("[INFO] Found %d forms for org %s (%s)", len(relevantForms), randomUser.ActiveOrg.Name, randomUser.ActiveOrg.Id)
body, err := json.Marshal(relevantForms)
if err != nil {
log.Printf("[WARNING] Failed form GET marshalling: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(`{"success": false}`))
return
}
resp.WriteHeader(200)
resp.Write(body)
}
func SendDeleteWorkflowRequest(childWorkflow Workflow, request *http.Request) error {
log.Printf("[INFO] Attempting to delete child workflow %s", childWorkflow.ID)
// Send a Delete request to the workflows
baseUrl := "https://shuffler.io"
if len(os.Getenv("BASE_URL")) > 0 {
baseUrl = os.Getenv("BASE_URL")
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s", baseUrl, childWorkflow.ID)
client := GetExternalClient(baseUrl)
req, err := http.NewRequest(
"DELETE",
fullUrl,
nil,
)
if err != nil {
log.Printf("[ERROR] Failed to delete child workflow %s: %s", childWorkflow.ID, err)
return err
}
// Look for Authorization
for key, values := range request.Header {
if len(values) > 0 {
req.Header.Add(key, values[0])
}
}
// Cookies
for _, cookie := range request.Cookies() {
req.AddCookie(cookie)
}
// Ensure it points correctly, and that you can only delete the ones you have access to
if len(childWorkflow.OrgId) > 0 {
req.Header.Add("Org-Id", childWorkflow.OrgId)
}
resp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Failed to delete child workflow %s: %s", childWorkflow.ID, err)
return err
}
if resp.StatusCode != 200 {
log.Printf("[ERROR] Failed to delete child workflow %s: %s", childWorkflow.ID, resp.Status)
return fmt.Errorf("Failed to delete child workflow %s: %s", childWorkflow.ID, resp.Status)
}
log.Printf("[INFO] Deleted child workflow %s. Resp: %s", childWorkflow.ID, string(resp.Status))
return nil
}
func NewTimeWindow(duration time.Duration) *TimeWindow {
return &TimeWindow{
Duration: duration,
Events: []time.Time{},
}
}
func (tw *TimeWindow) AddEvent(event time.Time) {
tw.mu.Lock()
defer tw.mu.Unlock()
tw.Events = append(tw.Events, event)
tw.cleanOldEvents(event)
}
func (tw *TimeWindow) CountEvents(now time.Time) int {
tw.mu.Lock()
defer tw.mu.Unlock()
tw.cleanOldEvents(now)
return len(tw.Events)
}
func (tw *TimeWindow) cleanOldEvents(now time.Time) {
cutoff := now.Add(-tw.Duration)
for len(tw.Events) > 0 && tw.Events[0].Before(cutoff) {
tw.Events = tw.Events[1:]
}
}
// Compute simple edit distance between two strings
func editDistance(a, b string) int {
n := len(a)
m := len(b)
dp := make([][]int, n+1)
for i := range dp {
dp[i] = make([]int, m+1)
dp[i][0] = i
}
for j := 0; j <= m; j++ {
dp[0][j] = j
}
for i := 1; i <= n; i++ {
for j := 1; j <= m; j++ {
if a[i-1] == b[j-1] {
dp[i][j] = dp[i-1][j-1]
} else {
dp[i][j] = min(dp[i-1][j-1], min(dp[i-1][j], dp[i][j-1])) + 1
}
}
}
return dp[n][m]
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
// SanitizeFuzzySubstring replaces any contiguous substring of haystack
// that is close to secret with "***"
func SanitizeFuzzySubstring(haystack, secret string, maxDistance int) string {
hayRunes := []rune(haystack)
hayLower := strings.ToLower(haystack)
secLower := strings.ToLower(secret)
secLen := len(secLower)
i := 0
for i < len(hayRunes) {
bestMatchStart := -1
bestMatchEnd := -1
bestDistance := maxDistance + 1
// Iterate through possible window sizes
for windowLen := secLen - maxDistance; windowLen <= secLen+maxDistance && i+windowLen <= len(hayLower); windowLen++ {
if windowLen <= 0 {
continue
}
window := hayLower[i : i+windowLen]
distance := editDistance(window, secLower)
// Found a closer match within the tolerance
if distance <= maxDistance && distance < bestDistance {
bestDistance = distance
bestMatchStart = i
bestMatchEnd = i + windowLen
}
}
// If a match was found, sanitize and jump the index past the sanitized section
if bestMatchStart != -1 {
for j := bestMatchStart; j < bestMatchEnd; j++ {
hayRunes[j] = '*'
}
i = bestMatchEnd
} else {
// No match, move to the next character
i++
}
}
return string(hayRunes)
}
// Basic cleanup.
func cleanupProtectedKeys(exec WorkflowExecution) WorkflowExecution {
// Basic checks to return
if os.Getenv("SHUFFLE_PROTECTED_CLEANUP_DISABLED") == "true" || project.Environment == "worker" {
return exec
}
// This doesn't matter as we are checking for 'sanitized' anyway
// This also makes it stop too early
//if exec.Status == "FINISHED" || exec.Status == "ABORTED" {
// return exec
//}
protectedKeys, _, err := GetAllCacheKeys(context.Background(), exec.ExecutionOrg, "protected", 100, "")
if err != nil {
//log.Printf("[ERROR] Failed getting protected keys for org %s: %s", exec.ExecutionOrg, err)
return exec
}
for resultKey, _ := range exec.Results {
if exec.Results[resultKey].Sanitized {
continue
}
if exec.Results[resultKey].Status != "FINISHED" && exec.Results[resultKey].Status != "SUCCESS" {
continue
}
for _, protectedKey := range protectedKeys {
if len(protectedKey.Value) <= 8 {
exec.Results[resultKey].Result = strings.ReplaceAll(exec.Results[resultKey].Result, protectedKey.Value, "***")
} else if len(protectedKey.Value) > 2000000 {
exec.Results[resultKey].Result = strings.ReplaceAll(exec.Results[resultKey].Result, protectedKey.Value, "***")
} else {
exec.Results[resultKey].Result = strings.ReplaceAll(exec.Results[resultKey].Result, protectedKey.Value, "***")
// FIXME: Should do more fuzzy sanitizing, but there are too
// many edgecases for it
//exec.Results[resultKey].Result = SanitizeFuzzySubstring(exec.Results[resultKey].Result, protectedKey.Value, 2)
}
}
exec.Results[resultKey].Sanitized = true
}
return exec
}
// Updates statuses in relevant areas according to what happened in the workflow run
func checkExecutionStatus(ctx context.Context, exec *WorkflowExecution) *WorkflowExecution {
// Check if this is already done
if exec.Status != "FINISHED" && exec.Status != "ABORTED" {
return exec
}
// FIXME: Skipping subexecs, as they are usually not relevant by themselves
/*
if len(exec.ExecutionParent) > 0 {
return exec
}
*/
// Create cache as to whether this has been ran in the last minute
cacheKey := fmt.Sprintf("validation_%s", exec.ExecutionId)
validationData, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(validationData.([]uint8))
err = json.Unmarshal(cacheData, &exec.Workflow.Validation)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling cache data for execution status: %s", err)
}
//log.Printf("\n\n[DEBUG][%s] Execution status already checked. Validation: %#v\n\n", exec.ExecutionId, exec.Workflow.Validation)
return exec
}
// FIXME: This is missing SKIPPED nodes that actually do run
// and may want to be counted due to checking conditions
amountFinished := 0
for _, res := range exec.Results {
if res.Status != "SKIPPED" {
amountFinished += 1
continue
}
}
IncrementCache(ctx, exec.ExecutionOrg, "app_executions", amountFinished)
go RunCacheCleanup(ctx, *exec)
//go RunIOCFinder(ctx, *exec)
//log.Printf("[DEBUG][%s] Running status fixing for workflow %#v to see if auth + workflow(s) are functional. Results: %d", exec.ExecutionId, exec.Workflow.ID, len(exec.Results))
orgId := exec.ExecutionOrg
allAuth, err := GetAllWorkflowAppAuth(ctx, orgId)
if err != nil {
log.Printf("[ERROR] Failed getting all auths for org during stat checks %s: %s", orgId, err)
return exec
}
workflow, err := GetWorkflow(ctx, exec.Workflow.ID, true)
if err != nil {
log.Printf("[WARNING] Failed getting workflow '%s': %s (exec status)", exec.Workflow.ID, err)
//workflow = &exec.Workflow
//return exec
}
// Make sure it only handles/keeps the relevant actions
// This helps us make sure we don't look into random actions that aren't directly connected
childNodes := FindChildNodes(exec.Workflow, exec.Start, []string{}, []string{})
newActions := []Action{}
for _, action := range workflow.Actions {
if exec.Start == action.ID {
newActions = append(newActions, action)
continue
}
if ArrayContains(childNodes, action.ID) {
newActions = append(newActions, action)
continue
}
}
originalActions := workflow.Actions
if len(newActions) > 0 {
workflow.Actions = newActions
}
if len(workflow.Actions) == 0 {
workflow.Actions = exec.Workflow.Actions
}
authenticationProblems := []ValidationProblem{}
handledAuth := []string{}
timenow := time.Now().Unix() * 1000
runtimeLocationName := ""
for _, action := range exec.Workflow.Actions {
if len(action.Environment) > 0 {
runtimeLocationName = action.Environment
break
}
}
//log.Printf("\n\n[DEBUG][%s] STARTING VALIDATION WITH %d results and %d actions\n\n", exec.ExecutionId, len(exec.Results), len(workflow.Actions))
for _, result := range exec.Results {
// FIXME: Skipping anything that outright fails right now
if result.Status == "SKIPPED" {
continue
}
if len(runtimeLocationName) == 0 && len(result.Action.Environment) > 0 {
runtimeLocationName = result.Action.Environment
}
found := false
foundAction := Action{}
for _, action := range workflow.Actions {
if action.ID != result.Action.ID {
continue
}
found = true
authRequired := false
for _, param := range action.Parameters {
// If authentication + has no value
if param.Configuration {
if len(param.Value) == 0 {
authRequired = true
}
}
}
// Check if this is an authentication action
if authRequired && action.AuthenticationId == "" {
// Check if authentication is required
authenticationProblems = append(authenticationProblems, ValidationProblem{
ActionId: action.ID,
AppId: action.AppID,
AppName: action.AppName,
Error: "No authentication specified",
Type: "authentication",
})
break
}
foundAction = action
break
}
if !found {
continue
}
if len(foundAction.AuthenticationId) > 0 && ArrayContains(handledAuth, foundAction.AuthenticationId) {
continue
}
// FIXME: try to make it a list of items first
listUnmarshalled := []HTTPOutput{}
err := json.Unmarshal([]byte(result.Result), &listUnmarshalled)
if len(listUnmarshalled) > 0 {
//log.Printf("[DEBUG] Unmarshal list success")
} else {
singleHttpItem := HTTPOutput{}
err := json.Unmarshal([]byte(result.Result), &singleHttpItem)
if err != nil {
//log.Printf("[WARNING] Failed unmarshalling http result for %s: %s", result.Action.Label, err)
//continue
} else {
listUnmarshalled = []HTTPOutput{singleHttpItem}
}
}
for _, unmarshalledHttp := range listUnmarshalled {
isValid := false
if unmarshalledHttp.Success == true {
if unmarshalledHttp.Status >= 200 && unmarshalledHttp.Status < 300 {
isValid = true
} else if unmarshalledHttp.Status != 0 {
validationProblem := ValidationProblem{
ActionId: foundAction.ID,
AppId: foundAction.AppID,
AppName: foundAction.AppName,
Error: fmt.Sprintf("Status %d for action '%s'. Are the fields correct?", unmarshalledHttp.Status, strings.ReplaceAll(foundAction.Label, "_", " ")),
Type: "configuration",
}
if unmarshalledHttp.Status == 401 {
validationProblem.Type = "authentication"
}
if unmarshalledHttp.Status == 403 {
validationProblem.Type = "authorization"
}
authenticationProblems = append(authenticationProblems, validationProblem)
break
}
} else {
if len(unmarshalledHttp.Reason) > 0 {
validationProblem := ValidationProblem{
ActionId: foundAction.ID,
AppId: foundAction.AppID,
AppName: foundAction.AppName,
Error: fmt.Sprintf("Action '%s' failed: '%s'", strings.ReplaceAll(foundAction.Label, "_", " "), unmarshalledHttp.Reason),
Type: "configuration",
}
authenticationProblems = append(authenticationProblems, validationProblem)
break
} else {
// Remove spaces and newlines, then check if it actually contains "success":false or not
formattedResult := strings.Replace(strings.Replace(strings.Replace(result.Result, " ", "", -1), "\n", "", -1), "\t", "", -1)
if !strings.Contains(formattedResult, `"success":false`) {
continue
}
validationProblem := ValidationProblem{
ActionId: foundAction.ID,
AppId: foundAction.AppID,
AppName: foundAction.AppName,
Error: "Success is false: Check node for more failure details",
Type: "configuration",
}
authenticationProblems = append(authenticationProblems, validationProblem)
break
}
// FIXME: What do we do here if there is no reason?
}
//log.Printf("\n\n\n[DEBUG][%s] Checking result for %s\n\n\n", exec.ExecutionId, result.Action.Label)
handledAuth = append(handledAuth, foundAction.AuthenticationId)
for _, auth := range allAuth {
if auth.Id != foundAction.AuthenticationId {
continue
}
authUpdated := false
// Check if the auth is still valid
if !isValid {
// Check if existing is valid or not
// if auth.Validation.V == false {
// //log.Printf("[DEBUG] Auth %s is already invalid", auth.Id)
if auth.Validation.Valid {
auth.Validation.Valid = false
authUpdated = true
}
// Making sure it's set once, with tests
if auth.Validation.ChangedAt == 0 {
authUpdated = true
}
} else {
// New is valid if here. If already valid, do nothing
if !auth.Validation.Valid {
auth.Validation.Valid = true
authUpdated = true
}
// Check if it is more than 10 days ago. If so, update again.
tenDaysMicroseconds := int64(432000000)
if timenow-auth.Validation.LastValid > tenDaysMicroseconds {
authUpdated = true
}
}
if authUpdated {
auth.Validation.ChangedAt = timenow
if auth.Validation.Valid {
auth.Validation.LastValid = timenow
}
auth.Validation.Environment = runtimeLocationName
auth.Validation.WorkflowId = workflow.ID
auth.Validation.ExecutionId = exec.ExecutionId
auth.Validation.NodeId = result.Action.ID
auth.Validation.ValidationRan = true
if len(auth.App.LargeImage) == 0 {
auth.App.LargeImage = result.Action.LargeImage
}
err = SetWorkflowAppAuthDatastore(ctx, auth, auth.Id)
if err != nil {
log.Printf("[ERROR] Failed updating auth at end of workflow run %s: %s", auth.Id, err)
} else {
log.Printf("[DEBUG] Updated auth %s for workflow %s", auth.Id, workflow.ID)
}
}
}
}
}
// FIXME: Check status from subflows as well
// Maybe subflows should update the parent?
// What if the subflow is a child of startnode, but didn't run?
// Then we just need a previous status..?
// SOMETHING has to run the update back to the parent to ensure
// subflows are accounted for
workflow.Validation.SubflowApps = []ValidationProblem{}
for _, trigger := range workflow.Triggers {
if trigger.TriggerType != "SUBFLOW" {
continue
}
if !ArrayContains(childNodes, trigger.ID) {
continue
}
// Replace with the apps of the subflow?
//log.Printf("\n\n\nSUBFLOW: %#v\n\n\n", trigger.ID)
foundWorkflow := ""
startNode := ""
_ = startNode
waitForResults := false
_ = waitForResults
for _, param := range trigger.Parameters {
if param.Name == "workflow" {
foundWorkflow = param.Value
}
if param.Name == "startnode" {
startNode = param.Value
}
if param.Name == "check_result" {
waitForResults = strings.ToLower(param.Value) == "true"
}
}
if foundWorkflow == "" {
continue
}
// Doing explicit execution IF it exists
foundExecutionIds := []string{}
for _, res := range exec.Results {
if res.Action.ID != trigger.ID {
continue
}
marshalledListData := []SubflowData{}
err := json.Unmarshal([]byte(res.Result), &marshalledListData)
if err != nil {
//log.Printf("[ERROR] Failed unmarshalling subflow data for %s: %s", res.Action.Label, err)
marshalledData := SubflowData{}
err := json.Unmarshal([]byte(res.Result), &marshalledData)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling subflow data for %s: %s", res.Action.Label, err)
//continue
} else {
marshalledListData = append(marshalledListData, marshalledData)
}
}
for _, marshalledData := range marshalledListData {
if marshalledData.Success == false {
//log.Printf("[DEBUG] Subflow %s failed to start", marshalledData.ExecutionId)
continue
}
foundExecutionIds = append(foundExecutionIds, marshalledData.ExecutionId)
}
break
}
//log.Printf("\n\n[DEBUG] Waiting for results. Execution IDs: %#v\n\n", foundExecutionIds)
appendedActionIds := []string{}
for _, execId := range foundExecutionIds {
subExec, err := GetWorkflowExecution(ctx, execId)
if err != nil {
log.Printf("[ERROR] Failed getting subflow execution %s for workflow %s: %s", execId, workflow.ID, err)
continue
}
if subExec.Status == "EXECUTING" {
// FIXME: Check based on the workflow itself instead
//log.Printf("[DEBUG] Subflow %s is still executing. Validation: %s", execId, subExec.Workflow.Validation.Valid)
// Loading the Workflows own validation in this case
oldWf, err := GetWorkflow(ctx, subExec.Workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed getting subflow %s for workflow %s: %s", subExec.Workflow.ID, workflow.ID, err)
} else {
subExec.Workflow = *oldWf
}
}
// Check validations
//log.Printf("[DEBUG] Subflow %s is finished. Validation: %#v. Validation.Errors: %d", execId, subExec.Workflow.Validation.Valid, len(subExec.Workflow.Validation.Errors))
if subExec.Workflow.Validation.Valid {
continue
}
for _, subProblem := range subExec.Workflow.Validation.Errors {
// We keep appending for each level
if ArrayContains(appendedActionIds, subProblem.ActionId) {
continue
}
appendedActionIds = append(appendedActionIds, subProblem.ActionId)
subProblem.Error = fmt.Sprintf("[SUBFLOW] %s", subProblem.Error)
subProblem.Type = "subflow_app"
workflow.Validation.SubflowApps = append(workflow.Validation.SubflowApps, subProblem)
}
if len(subExec.Workflow.Validation.SubflowApps) > 0 {
for _, subProblem := range subExec.Workflow.Validation.SubflowApps {
// We keep appending for each level
if ArrayContains(appendedActionIds, subProblem.ActionId) {
continue
}
appendedActionIds = append(appendedActionIds, subProblem.ActionId)
subProblem.Type = fmt.Sprintf("sub_%s", subProblem.Type)
if len(subProblem.Type) > 20 {
subProblem.Type = subProblem.Type[:20] + "_app"
}
workflow.Validation.SubflowApps = append(workflow.Validation.SubflowApps, subProblem)
}
}
}
}
// Dedup subflowapps
newApps := []ValidationProblem{}
for _, app := range workflow.Validation.SubflowApps {
found := false
for _, newApp := range newApps {
if newApp.ActionId == app.ActionId {
found = true
break
}
}
if !found {
newApps = append(newApps, app)
}
}
workflow.Validation.SubflowApps = newApps
workflowChanged := false
workflow.Validation.Errors = authenticationProblems
if len(workflow.Validation.Errors) > 0 {
workflow.Validation.Valid = false
} else {
workflow.Validation.Valid = true
}
// FIXME: Set the right stuff for the workflow here as well
workflow.Validation.ChangedAt = timenow
if workflow.Validation.Valid {
workflow.Validation.LastValid = timenow
workflow.Validation.ExecutionId = exec.ExecutionId
}
workflow.Validation.TotalProblems = len(workflow.Validation.Errors) + len(workflow.Validation.SubflowApps)
//log.Printf("\n\n\nVALIDATION RUNNING\n\n\n")
// Updating the workflow to show the right status every time for now
workflowChanged = true
workflow.Validation.Environment = runtimeLocationName
workflow.Validation.ValidationRan = true
workflow.Validation.ExecutionId = exec.ExecutionId
if workflowChanged {
workflow.Actions = originalActions
// This causes too many writes and can't be handled at scale. Removing for now. Only setting cache.
/*
// FIXME: Even removing cache due to possibility of workflow override if an execution is finishing after a users' save. Also fails with delays. For now, using validation_workflow_%s to handle it all
*/
}
exec.Workflow.Validation.NotificationsCreated = exec.NotificationsCreated
exec.Workflow.Validation = workflow.Validation
marshalledValidation, err := json.Marshal(workflow.Validation)
if err != nil {
return exec
}
// Force them to work without parent context management
backgroundContext := context.Background()
SetCache(backgroundContext, fmt.Sprintf("validation_workflow_%s", workflow.ID), marshalledValidation, 1440)
SetCache(backgroundContext, cacheKey, marshalledValidation, 120)
// ALWAYS have correct exec id for current execution, but not always in workflow
//log.Printf("\n\n[DEBUG][%s] Set workflow validation (%d) to '%s'\n\n", exec.ExecutionId, len(workflow.Validation.Errors), marshalledValidation)
return exec
}
func HandleDatastoreCategoryConfig(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
// Checking if it's a special region. All user-specific requests should
ctx := GetContext(request)
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get org: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Only admins can access this endpoint"}`))
return
}
categoryUpdate := DatastoreCategoryUpdate{}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[ERROR] Failed reading body in datastore category config: %s", err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
err = json.Unmarshal(body, &categoryUpdate)
if err != nil {
log.Printf("[ERROR] Failed unmarshalling body in datastore category config: %s", err)
resp.WriteHeader(http.StatusBadRequest)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
if len(categoryUpdate.Category) == 0 || strings.ToLower(categoryUpdate.Category) == "default" {
categoryUpdate.Category = ""
}
// Validate input - especially for workflows
for automationId, automation := range categoryUpdate.Automations {
if len(automation.Name) == 0 {
continue
}
// Don't want to do this either just in case they have something configured, but unused
/*
if automation.Enabled != true {
continue
}
*/
if strings.ToLower(automation.Name) == "run ai agent" {
for optionIndex, option := range automation.Options {
if option.Key == "" {
automation.Options[optionIndex].Key = "action"
}
}
} else if strings.ToLower(automation.Name) == "run workflow" {
foundWorkflowIds := ""
foundWorkflowIdIndex := -1
for optionIndex, option := range automation.Options {
if option.Key == "workflow_id" {
foundWorkflowIds = option.Value
foundWorkflowIdIndex = optionIndex
break
}
}
newWorkflows := []string{}
for _, workflowId := range strings.Split(foundWorkflowIds, ",") {
if len(workflowId) == 0 {
continue
}
wf, err := GetWorkflow(ctx, strings.TrimSpace(workflowId))
if err != nil {
log.Printf("[WARNING] Failed getting workflow '%s' for automation %d: %s", workflowId, automationId, err)
continue
}
if wf.OrgId != user.ActiveOrg.Id {
continue
}
newWorkflows = append(newWorkflows, workflowId)
}
if foundWorkflowIdIndex != -1 {
categoryUpdate.Automations[automationId].Options[foundWorkflowIdIndex].Value = strings.Join(newWorkflows, ",")
}
}
}
if categoryUpdate.Settings.Timeout < 60 {
categoryUpdate.Settings.Timeout = 0
} else if categoryUpdate.Settings.Timeout > 2147483647 {
categoryUpdate.Settings.Timeout = 0
}
categoryUpdate.OrgId = user.ActiveOrg.Id
err = SetDatastoreCategoryConfig(ctx, categoryUpdate)
if err != nil {
log.Printf("[ERROR] Failed setting category config: %s", err)
resp.WriteHeader(http.StatusInternalServerError)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
resp.WriteHeader(http.StatusOK)
resp.Write([]byte(`{"success": true}`))
}
func startSchedule(trigger Trigger, authorization string, workflow Workflow) error {
baseUrl := "https://shuffler.io"
if os.Getenv("BASE_URL") != "" {
baseUrl = os.Getenv("BASE_URL")
}
if len(os.Getenv("SHUFFLE_CLOUDRUN_URL")) > 0 {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
scheduleUrl := fmt.Sprintf("%s/api/v1/workflows/%s/schedule", baseUrl, workflow.ID)
// POST request to it
// Every 24 hours in cron
foundFrequency := "0 0 * * *"
for _, param := range trigger.Parameters {
if param.Name == "cron" && len(param.Value) > 2 {
foundFrequency = param.Value
}
}
scheduleRequest := Schedule{
Name: "Schedule",
Frequency: foundFrequency,
ExecutionArgument: "Automatically configured by Shuffle",
Environment: trigger.Environment,
Id: trigger.ID,
Start: workflow.Start,
}
parsedBody, err := json.Marshal(scheduleRequest)
if err != nil {
log.Printf("[ERROR] Failed marshalling schedule request: %s", err)
return err
}
// Send post request
client := GetExternalClient(baseUrl)
req, err := http.NewRequest(
"POST",
scheduleUrl,
bytes.NewBuffer(parsedBody),
)
if err != nil {
return err
}
// Add headers
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
if len(workflow.OrgId) > 0 {
req.Header.Add("Org-Id", workflow.OrgId)
}
// Add content type
req.Header.Add("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Failed to start schedule for workflow %s: %s", workflow.ID, err)
return err
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
body = []byte{}
}
if resp.StatusCode != 200 {
log.Printf("[ERROR] Failed to start schedule for workflow %s: %s. Body: %s", workflow.ID, resp.Status, string(body))
return errors.New(fmt.Sprintf("Failed to start schedule for workflow %s: %s", workflow.ID, resp.Status))
}
return nil
}
// getPrioritisedAppActions returns actions for an app, prioritised by most
// likely to be used.
// Simple sorting:
// 1. Categorised & Labeled actions
// 2. Most used actions across the platform
// 3. SHOULD BE MORE :))
func getPrioritisedAppActions(ctx context.Context, inputApp string, maxAmount int) []WorkflowAppAction {
returnActions := []WorkflowAppAction{}
appName := inputApp
foundApp := &WorkflowApp{}
appId := ""
var err error
if debug {
log.Printf("[DEBUG] Getting prioritised app actions for '%s'", inputApp)
}
if strings.Contains(inputApp, ":") || len(inputApp) == 32 {
appnamesplit := strings.Split(inputApp, ":")
appId = appnamesplit[0]
if len(appId) != 32 {
appId = ""
}
}
if len(appId) == 32 {
foundApp, err = GetApp(ctx, appId, User{}, false)
if err != nil {
log.Printf("[ERROR] Failed getting app %s for prioritised actions: %s", appId, err)
return returnActions
}
}
if foundApp.ID == "" && len(appName) > 0 {
log.Printf("[ERROR] Should find app + actions based on name (not implemented): %#v", appName)
}
for _, action := range foundApp.Actions {
if action.Name == "custom_action" {
continue
}
if len(action.CategoryLabel) > 0 {
//log.Printf("ACTION TAG: %#v => %#v", action.Label, action.CategoryLabel)
returnActions = append(returnActions, action)
continue
}
if len(returnActions) >= maxAmount {
break
}
}
if len(returnActions) <= maxAmount {
for _, action := range foundApp.Actions {
if len(returnActions) >= maxAmount {
break
}
if action.Name == "custom_action" {
continue
}
returnActions = append(returnActions, action)
}
}
if debug {
log.Printf("[DEBUG] Found %d priority actions for app %#v", len(returnActions), inputApp)
}
return returnActions
}
func GetDockerClient() (*dockerclient.Client, string, error) {
ctx := context.Background()
dockerApiVersion := os.Getenv("DOCKER_API_VERSION")
cli, err := dockerclient.NewEnvClient()
if err != nil {
return nil, dockerApiVersion, err
}
_, err = cli.Info(ctx)
if err == nil {
return cli, dockerApiVersion, nil
}
if strings.Contains(strings.ToLower(err.Error()), strings.ToLower("Minimum supported API version is")) {
re := regexp.MustCompile(`(?i)minimum supported api version is ([0-9\.]+)`)
match := re.FindStringSubmatch(err.Error())
if len(match) == 2 {
required := match[1]
os.Setenv("DOCKER_API_VERSION", required)
cli, err = dockerclient.NewEnvClient()
if err == nil {
dockerApiVersion = required
_, err = cli.Info(ctx)
return cli, dockerApiVersion, err
}
}
}
if strings.Contains(strings.ToLower(err.Error()), strings.ToLower("Maximum supported API version is")) {
re := regexp.MustCompile(`(?i)maximum supported api version is ([0-9\.]+)`)
match := re.FindStringSubmatch(err.Error())
if len(match) == 2 {
required := match[1]
os.Setenv("DOCKER_API_VERSION", required)
cli, err = dockerclient.NewEnvClient()
if err == nil {
dockerApiVersion = required
_, err = cli.Info(ctx)
return cli, dockerApiVersion, err
}
}
}
return cli, dockerApiVersion, err
}
// Syncs content between openapi and shuffle apps
// This is due to the generative growth nature of our apps
func syncAppContentLabels(ctx context.Context, id string, api *ParsedOpenApi) *ParsedOpenApi {
/*
// This doesn't seem to work, and isn't really in use
if api.Success == false {
if debug {
log.Printf("[DEBUG] Failed to load openapi %#v. Success false.", id)
}
return api
}
*/
// Get the app
app, err := GetApp(ctx, id, User{}, false)
if err != nil {
return api
}
if debug {
log.Printf("[DEBUG] Syncing app content labels for app %s", app.ID)
}
swaggerLoader := openapi3.NewSwaggerLoader()
swaggerLoader.IsExternalRefsAllowed = true
swagger, err := swaggerLoader.LoadSwaggerFromData([]byte(api.Body))
if err != nil {
log.Printf("[ERROR] Failed to load swagger during fix for ID %#v", id)
return api
}
openapiChanged := false
appChanged := false
for pathIndex, pathItem := range swagger.Paths {
if pathItem == nil {
continue
}
ops := map[string]*openapi3.Operation{
"GET": pathItem.Get,
"POST": pathItem.Post,
"PUT": pathItem.Put,
"DELETE": pathItem.Delete,
"PATCH": pathItem.Patch,
"HEAD": pathItem.Head,
"OPTIONS": pathItem.Options,
"TRACE": pathItem.Trace,
}
for method, op := range ops {
if op == nil {
continue
}
method := strings.ToLower(method)
parsedOpId := strings.ToLower(op.OperationID)
if !strings.HasPrefix(parsedOpId, method) {
parsedOpId = fmt.Sprintf("%s_%s", method, strings.ToLower(op.OperationID))
}
found := false
for actionIndex, action := range app.Actions {
//parsedActionName := fmt.Sprintf("%s_%s", method, action.Name)
parsedActionName := fmt.Sprintf("%s", action.Name)
if !strings.HasPrefix(parsedActionName, method) {
parsedActionName = fmt.Sprintf("%s_%s", method, strings.ToLower(parsedActionName))
}
if parsedActionName != parsedOpId {
continue
}
openapiLabels := []string{}
if methodLabels, ok := op.ExtensionProps.Extensions["x-label"]; ok {
labels := []string{}
j, err := json.Marshal(&methodLabels)
if err == nil {
err = json.Unmarshal(j, &labels)
if err != nil {
//log.Printf("[ERROR] Failed unmarshalling labels during sync for action %s (1): %s. Value: %#v\n", action.Name, err, j)
label := ""
err = json.Unmarshal(j, &label)
if err == nil {
labels = strings.Split(label, ",")
}
}
} else {
log.Printf("[ERROR] Failed marshalling labels during sync for action %s (2): %s\n", action.Name, err)
}
if len(labels) > 0 {
openapiLabels = labels
}
}
if len(openapiLabels) != len(action.CategoryLabel) {
if debug {
log.Printf("[DEBUG] APP DIFF (%s): %#v vs %#v", app.ID, openapiLabels, action.CategoryLabel)
}
seen := make(map[string]int)
for labelIndex, openapiLabel := range openapiLabels {
if strings.ReplaceAll(strings.ToLower(openapiLabel), " ", "_") == "no_label" {
openapiChanged = true
openapiLabel = "No Label"
openapiLabels[labelIndex] = openapiLabel
}
if strings.HasPrefix(openapiLabel, "[") && strings.HasSuffix(openapiLabel, "]") && len(openapiLabel) > 2 {
openapiChanged = true
openapiLabel = openapiLabel[1 : len(openapiLabel)-1]
openapiLabels[labelIndex] = openapiLabel
}
seen[openapiLabel] = 1
}
for labelIndex, actionLabel := range action.CategoryLabel {
if strings.ReplaceAll(strings.ToLower(actionLabel), " ", "_") == "no_label" {
appChanged = true
actionLabel = "No Label"
action.CategoryLabel[labelIndex] = actionLabel
}
if strings.HasPrefix(actionLabel, "[") && strings.HasSuffix(actionLabel, "]") && len(actionLabel) > 2 {
appChanged = true
actionLabel = actionLabel[1 : len(actionLabel)-1]
action.CategoryLabel[labelIndex] = actionLabel
}
if _, ok := seen[actionLabel]; ok {
seen[actionLabel] = 0
} else {
seen[actionLabel] = 2
}
}
for key, value := range seen {
if strings.ReplaceAll(strings.ToLower(key), " ", "_") == "no_label" {
key = "No Label"
}
switch value {
case 1:
openapiChanged = true
action.CategoryLabel = append(action.CategoryLabel, key)
app.Actions[actionIndex] = action
case 2:
appChanged = true
openapiLabels = append(openapiLabels, key)
// Update in openapi
op.ExtensionProps.Extensions["x-label"] = openapiLabels
swagger.Paths[pathIndex].Get = pathItem.Get
}
}
}
found = true
break
}
if !found {
log.Printf("[ERROR] Failed to find operation %s. May be missing sync between app <-> openapi. ID: %s", parsedOpId, id)
}
}
}
if openapiChanged {
api.Success = true
marshalledSwagger, err := json.Marshal(swagger)
if err != nil {
log.Printf("[ERROR] Failed marshalling swagger after sync for ID %#v", id)
} else {
api.Body = string(marshalledSwagger)
SetOpenApiDatastore(ctx, app.ID, *api)
}
}
if appChanged {
SetWorkflowAppDatastore(ctx, *app, app.ID)
}
return api
}
// FuzzyHashBody collapses tiny differences in numbers and short strings
func FuzzyHashBody(body []byte) uint64 {
hasher := fnv.New64a()
i := 0
for i < len(body) {
b := body[i]
switch {
// Skip whitespace
case unicode.IsSpace(rune(b)):
i++
// Numbers → bucket as "NUM"
case b >= '0' && b <= '9':
hasher.Write([]byte("NUM"))
// skip the full number
for i < len(body) && body[i] >= '0' && body[i] <= '9' {
i++
}
// Letters → bucket as uppercase
case unicode.IsLetter(rune(b)):
start := i
for i < len(body) && unicode.IsLetter(rune(body[i])) {
i++
}
// convert to uppercase token
token := body[start:i]
for _, c := range token {
if c >= 'a' && c <= 'z' {
c = c - 'a' + 'A'
}
hasher.Write([]byte{byte(c)})
}
// Everything else → keep as-is
default:
hasher.Write([]byte{b})
i++
}
}
return hasher.Sum64()
}
// Checks whether e.g. a workflow is calling itself with VERY similar details.
// URL MUST be identical, but body can vary slightly and still match.
// Implementations (cloud):
// - /workflow/{workflowId}/run
// - /apps/{appId}/run
// - /hooks/{webhookId}
func IsExecutionRecursion(ctx context.Context, request *http.Request, body []byte) bool {
// May not have enough details to know without a body (?)
if len(body) == 0 {
return false
}
urlMd5 := Md5sum([]byte(request.URL.String()))
// Hashes the body into "buckets" that look for slight similarities
// The main point is avoiding replicas with deviations like timestamps
hash1 := FuzzyHashBody(body)
//cacheKey := fmt.Sprintf("%s_%s", urlMd5, hash1)
cacheKey := fmt.Sprintf("hash_%s_%d", urlMd5, hash1)
cache, err := GetCache(ctx, cacheKey)
if err != nil {
SetCache(ctx, cacheKey, []byte("1"), 1)
return false
}
foundNumber := 0
cacheData := string(cache.([]uint8))
if n, err := strconv.Atoi(cacheData); err == nil {
foundNumber = n
}
if foundNumber > 0 {
foundNumber += 1
} else {
foundNumber = 1
}
// Controllable
defaultRecursionDepth := 5
maxRecursionDepthInt := defaultRecursionDepth
maxRecursionDepth := os.Getenv("SHUFFLE_MAX_RECURSION_DEPTH")
if maxRecursionDepth == "" {
maxRecursionDepthInt, err = strconv.Atoi(maxRecursionDepth)
if err != nil {
maxRecursionDepthInt = defaultRecursionDepth
}
}
if maxRecursionDepthInt < 3 {
maxRecursionDepthInt = 3
}
// This has monitoring on it and should ideally NEVER happen
if foundNumber > maxRecursionDepthInt {
log.Printf("[ERROR] Detected potential recursion for URL %s. Hash: %d", request.URL.String(), hash1)
return true
}
SetCache(ctx, cacheKey, []byte(strconv.Itoa(foundNumber)), 1)
return false
}
// normalizeToMs coerces an ActionResult timestamp to milliseconds, regardless of
// whether it was stored as seconds, ms, microseconds, or nanoseconds. Mirrors the
// 10/13/19-digit handling in FixActionResultOutput, plus a 16-digit branch for
// time.Now().UnixMicro() values written by db-connector.go and parts of shared.go.
func normalizeToMs(ts int64) int64 {
switch len(strconv.FormatInt(ts, 10)) {
case 10:
return ts * 1000
case 13:
return ts
case 16:
return ts / 1000
case 19:
return ts / 1000000
default:
return ts
}
}
// ValidateExecutionChronology checks if actions in a workflow execution started
// before all their parents completed. Returns violations with parent-child timing mismatches.
func ValidateExecutionChronology(ctx context.Context, execution *WorkflowExecution) []ExecutionChronologyViolation {
var violations []ExecutionChronologyViolation
if execution == nil || len(execution.Results) == 0 {
return violations
}
// Build parent map: destID -> []srcID (same logic as CheckNextActions, skip decorators)
parents := make(map[string][]string)
for _, branch := range execution.Workflow.Branches {
if branch.Decorator {
continue
}
parents[branch.DestinationID] = append(parents[branch.DestinationID], branch.SourceID)
}
// Index results by action ID
resultsByID := make(map[string]ActionResult)
for _, result := range execution.Results {
if result.Action.ID != "" {
resultsByID[result.Action.ID] = result
}
}
// Check each executed action against its parents
for _, result := range execution.Results {
if result.Status == "SKIPPED" || result.Action.ID == "" || result.StartedAt == 0 {
continue
}
for _, parentID := range parents[result.Action.ID] {
parentResult, ok := resultsByID[parentID]
if !ok || parentResult.Status == "SKIPPED" || parentResult.CompletedAt == 0 {
// parent didn't run, was skipped, or is a trigger with no timing — not a dependency
continue
}
childMs := normalizeToMs(result.StartedAt)
parentMs := normalizeToMs(parentResult.CompletedAt)
if childMs < parentMs {
gapMs := parentMs - childMs
violations = append(violations, ExecutionChronologyViolation{
ActionID: result.Action.ID,
ActionLabel: result.Action.Label,
ParentID: parentID,
ActionStart: childMs,
ParentEnd: parentMs,
GapMs: gapMs,
})
log.Printf("[WARNING][%s] Ordering violation: %s started %.2fs before parent %s (%s) completed",
execution.ExecutionId, result.Action.Label,
float64(gapMs)/1000.0, parentID, parentResult.Action.Label)
}
}
}
return violations
}
func listProcessesWindows() ([]ProcessInfo, error) {
return collect()
}
func listProcessesDarwin() ([]ProcessInfo, error) {
return collect()
}
func listProcessesLinux() ([]ProcessInfo, error) {
return collect()
}
type cacheEntry struct {
hash string
mtime time.Time
size int64
}
var (
hashCache = make(map[string]cacheEntry)
hashCacheMu sync.Mutex
)
// cachedHashFile returns the SHA256 of the file at path.
// It only re-hashes if the file's mtime or size has changed since last call.
func cachedHashFile(path string) string {
if path == "" {
return ""
}
info, err := os.Stat(path)
if err != nil {
return ""
}
mtime := info.ModTime()
size := info.Size()
hashCacheMu.Lock()
entry, ok := hashCache[path]
hashCacheMu.Unlock()
if ok && entry.mtime.Equal(mtime) && entry.size == size {
return entry.hash
}
// Cache miss or file changed — hash it.
hash := hashFile(path)
if hash == "" {
return ""
}
hashCacheMu.Lock()
hashCache[path] = cacheEntry{hash: hash, mtime: mtime, size: size}
hashCacheMu.Unlock()
return hash
}
// hashFile computes the SHA256 of a file by streaming it —
// large binaries never fully land in memory.
func hashFile(path string) string {
f, err := os.Open(path)
if err != nil {
return ""
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return ""
}
return hex.EncodeToString(h.Sum(nil))
}
func scrubArgs(args []string) []string {
if len(args) == 0 {
return args
}
out := make([]string, len(args))
copy(out, args)
for i, arg := range out {
// Style 1: --flag=value or -f=value
if eq := indexByte(arg, '='); eq >= 0 {
key := arg[:eq]
if isSecretKey(key) {
out[i] = key + "=[REDACTED]"
}
continue
}
// Style 2/3: --flag value or -f value — redact the next element.
if isSecretKey(arg) && i+1 < len(out) {
out[i+1] = "[REDACTED]"
}
}
return out
}
var secretKeywords = []string{
"token",
"secret",
"password",
"passwd",
"apikey",
"api_key",
"api-key",
"auth",
"credential",
"private_key",
"private-key",
"access_key",
"access-key",
"signing_key",
"signing-key",
}
// isSecretKey returns true if the flag name contains a secret keyword.
func isSecretKey(flag string) bool {
// Strip leading dashes so "--api-key" and "api-key" both match.
lower := strings.ToLower(strings.TrimLeft(flag, "-"))
for _, kw := range secretKeywords {
if strings.Contains(lower, kw) {
return true
}
}
return false
}
// indexByte returns the index of the first occurrence of c in s, or -1.
// Using this instead of strings.IndexByte to avoid an extra import.
func indexByte(s string, c byte) int {
for i := 0; i < len(s); i++ {
if s[i] == c {
return i
}
}
return -1
}
// collect is identical on both platforms — gopsutil handles the syscall difference.
func collect() ([]ProcessInfo, error) {
procs, err := process.Processes()
if err != nil {
return nil, fmt.Errorf("listing processes: %w", err)
}
out := make([]ProcessInfo, 0, len(procs))
for _, p := range procs {
ppid, err := p.Ppid()
if err != nil {
ppid = 0
}
tty, err := p.Terminal() // "" if no controlling terminal
if err != nil {
tty = ""
}
cmd, err := p.Name() // argv[0] basename
if err != nil {
cmd = ""
}
user, err := p.Username()
if err != nil {
user = ""
}
exePath, err := p.Exe()
if err != nil {
exePath = ""
}
// kernel threads and SIP-protected processes.
args, err := p.CmdlineSlice()
if err != nil {
args = nil
}
args = scrubArgs(args)
createdAt, err := p.CreateTime()
if err != nil {
createdAt = 0
}
out = append(out, ProcessInfo{
PID: p.Pid,
PPID: ppid,
TTY: tty,
CommandLine: cmd,
User: user,
Args: args,
CreationTime: createdAt,
ExePath: exePath,
// Hash the binary on disk. Note: this is the file at rest, not the
// in-memory image — a binary replaced after launch won't be caught here.
SHA256: cachedHashFile(exePath),
})
}
if debug {
log.Printf("[INFO] Found %d processes", len(out))
}
return out, nil
}
// ListProcesses returns all running processes.
// On macOS this calls sysctl kern.proc under the hood.
// On Linux this reads /proc.
func ListProcesses() ([]ProcessInfo, error) {
switch runtime.GOOS {
case "darwin":
return listProcessesDarwin()
case "linux":
return listProcessesLinux()
case "windows":
return listProcessesWindows()
default:
return nil, fmt.Errorf("unsupported platform: %s", runtime.GOOS)
}
}