marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4934f92ddbcb3d8b807da6a4303685494dd5c984
shuffle-cracked/backend/go-app/shuffle-shared/cloudSync.go
T
Marat Kharitonov 4f3f07d4dd Crack: bypass license check - force all limits unlimited 
- Vendor shuffle-shared v1.2.51 as backend/go-app/shuffle-shared
- Add replace directive in go.mod to use the local moduled copy
- In HandleCheckLicense, force org.Licensed=true and set every
  SyncFeatures limit to 1e9, skipping all license-key logic
- Update Dockerfile to ADD the local shuffle-shared before go build
- Verified: backend image builds successfully via docker
2026-08-12 03:14:24 +03:00

3655 lines
113 KiB
Go
Raw Blame History

package shuffle
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"strconv"
"sync"
"encoding/base64"
//"github.com/algolia/algoliasearch-client-go/v3/algolia/opt"
"github.com/algolia/algoliasearch-client-go/v3/algolia/search"
"github.com/frikky/schemaless"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
//"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
"github.com/go-git/go-git/v5/plumbing/transport"
gitHttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
uuid "github.com/satori/go.uuid"
)
func executeCloudAction(action CloudSyncJob, apikey string) error {
data, err := json.Marshal(action)
if err != nil {
log.Printf("Failed cloud webhook action marshalling: %s", err)
return err
}
client := &http.Client{}
syncUrl := fmt.Sprintf("https://shuffler.io/api/v1/cloud/sync/handle_action")
req, err := http.NewRequest(
"POST",
syncUrl,
bytes.NewBuffer(data),
)
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
newresp, err := client.Do(req)
if err != nil {
return err
}
defer newresp.Body.Close()
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
return err
}
type Result struct {
Success bool `json:"success"`
Reason string `json:"reason"`
}
//log.Printf("Data: %s", string(respBody))
responseData := Result{}
err = json.Unmarshal(respBody, &responseData)
if err != nil {
return err
}
if !responseData.Success {
return errors.New(fmt.Sprintf("Cloud error from Shuffler: %s", responseData.Reason))
}
return nil
}
func HandleAlgoliaAppSearch(ctx context.Context, appname string) (AlgoliaSearchApp, error) {
cacheTimer := int32(300)
normalizedAppName := strings.TrimSpace(strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(appname, "_", " "), " ", "_")))
cacheKey := fmt.Sprintf("appsearch_%s", normalizedAppName)
cache, err := GetCache(ctx, cacheKey)
if err == nil {
if cacheData, ok := cache.([]byte); ok {
var cachedApp AlgoliaSearchApp
err = json.Unmarshal(cacheData, &cachedApp)
if err == nil {
return cachedApp, nil
}
log.Printf("[ERROR] Failed unmarshalling cached app search data in Handle algolia app search: %s", err)
}
}
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
// Fallback to default Algolia keys
if len(algoliaSecret) == 0 {
algoliaClient = "JNSS5CFDZZ"
algoliaSecret = os.Getenv("ALGOLIA_PUBLICKEY")
}
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[ERROR] ALGOLIA_CLIENT and ALGOLIA_SECRET/ALGOLIA_SECRET not defined (app discovery)")
return AlgoliaSearchApp{}, errors.New("Algolia keys not defined")
}
returnApp := AlgoliaSearchApp{}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("appsearch")
appname = strings.TrimSpace(strings.ToLower(strings.Replace(strings.Replace(appname, "_", " ", -1), "-", " ", -1)))
res, err := algoliaIndex.Search(appname)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia (%s): %s", appname, err)
appData, err := json.Marshal(returnApp)
if err == nil {
SetCache(ctx, cacheKey, appData, cacheTimer)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (3): %s", err)
}
return returnApp, err
}
var newRecords []AlgoliaSearchApp
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia: %s", err)
appData, err := json.Marshal(returnApp)
if err == nil {
SetCache(ctx, cacheKey, appData, cacheTimer)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (4): %s", err)
}
return returnApp, err
}
if debug {
log.Printf("[DEBUG] Got %d hits matching appname '%s'", len(newRecords), appname)
}
for _, newRecord := range newRecords {
newApp := strings.TrimSpace(strings.ToLower(strings.Replace(strings.Replace(newRecord.Name, "_", " ", -1), "-", " ", -1)))
if newApp == appname || newRecord.ObjectID == appname {
//return newRecord.ObjectID, nil
appData, err := json.Marshal(newRecord)
if err == nil {
SetCache(ctx, cacheKey, appData, cacheTimer)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (5): %s", err)
}
return newRecord, nil
}
}
// Second try with contains
for _, newRecord := range newRecords {
newApp := strings.TrimSpace(strings.ToLower(strings.Replace(newRecord.Name, "_", " ", -1)))
if strings.Contains(newApp, appname) {
appData, err := json.Marshal(newRecord)
if err == nil {
SetCache(ctx, cacheKey, appData, cacheTimer)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (6): %s", err)
}
return newRecord, nil
}
}
appData, err := json.Marshal(returnApp)
if err == nil {
SetCache(ctx, cacheKey, appData, cacheTimer)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (7): %s", err)
}
return returnApp, nil
}
func HandleAlgoliaWorkflowSearchByApp(ctx context.Context, appname string) ([]AlgoliaSearchWorkflow, error) {
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return []AlgoliaSearchWorkflow{}, errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("workflows")
appSearch := fmt.Sprintf("%s", appname)
res, err := algoliaIndex.Search(appSearch)
if err != nil {
log.Printf("[WARNING] Failed app searching Algolia for creators: %s", err)
return []AlgoliaSearchWorkflow{}, err
}
var newRecords []AlgoliaSearchWorkflow
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia with app creators: %s", err)
return []AlgoliaSearchWorkflow{}, err
}
//log.Printf("[INFO] Algolia hits for %s: %d", appSearch, len(newRecords))
allRecords := []AlgoliaSearchWorkflow{}
for _, newRecord := range newRecords {
allRecords = append(allRecords, newRecord)
}
return allRecords, nil
}
func HandleAlgoliaWorkflowSearchByUser(ctx context.Context, userId string) ([]AlgoliaSearchWorkflow, error) {
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return []AlgoliaSearchWorkflow{}, errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("workflows")
appSearch := fmt.Sprintf("%s", userId)
res, err := algoliaIndex.Search(appSearch)
if err != nil {
log.Printf("[WARNING] Failed app searching Algolia for creators: %s", err)
return []AlgoliaSearchWorkflow{}, err
}
var newRecords []AlgoliaSearchWorkflow
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia with app creators: %s", err)
return []AlgoliaSearchWorkflow{}, err
}
//log.Printf("[INFO] Algolia hits for %s: %d", appSearch, len(newRecords))
allRecords := []AlgoliaSearchWorkflow{}
for _, newRecord := range newRecords {
allRecords = append(allRecords, newRecord)
}
return allRecords, nil
}
func HandleAlgoliaAppSearchByUser(ctx context.Context, userId string) ([]AlgoliaSearchApp, error) {
cacheKey := fmt.Sprintf("appsearch_user_%s", userId)
cache, err := GetCache(ctx, cacheKey)
if err == nil {
if cacheData, ok := cache.([]byte); ok {
var cachedApp []AlgoliaSearchApp
err = json.Unmarshal(cacheData, &cachedApp)
if err == nil {
return cachedApp, nil
}
log.Printf("[ERROR] Failed unmarshalling cached app search data in Handle algolia app search for user (%s): %s", cacheKey, err)
}
}
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaSecret) == 0 {
algoliaClient = "JNSS5CFDZZ"
algoliaSecret = os.Getenv("ALGOLIA_PUBLICKEY")
}
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return []AlgoliaSearchApp{}, errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("appsearch")
returnApps := []AlgoliaSearchApp{}
appSearch := fmt.Sprintf("%s", userId)
res, err := algoliaIndex.Search(appSearch)
if err != nil {
log.Printf("[ERROR] Failed app searching Algolia for creators (%s): %s", appSearch, err)
appData, err := json.Marshal(returnApps)
if err == nil {
SetCache(ctx, cacheKey, appData, 30)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (8): %s", err)
}
return returnApps, err
}
var newRecords []AlgoliaSearchApp
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[ERROR] Failed unmarshaling from Algolia with app creators: %s", err)
appData, err := json.Marshal(returnApps)
if err == nil {
SetCache(ctx, cacheKey, appData, 30)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (9): %s", err)
}
return returnApps, err
}
for _, newRecord := range newRecords {
newAppName := strings.TrimSpace(strings.Replace(newRecord.Name, "_", " ", -1))
newRecord.Name = newAppName
returnApps = append(returnApps, newRecord)
}
appData, err := json.Marshal(returnApps)
if err == nil {
SetCache(ctx, cacheKey, appData, 30)
} else {
log.Printf("[ERROR] Failed to marshal Algolia result in handle aloglia search (10): %s", err)
}
return returnApps, nil
}
func HandleAlgoliaCreatorSearch(ctx context.Context, username string) (AlgoliaSearchCreator, error) {
tmpUsername, err := url.QueryUnescape(username)
if err == nil {
username = tmpUsername
}
if strings.HasPrefix(username, "@") {
username = strings.Replace(username, "@", "", 1)
}
username = strings.ToLower(strings.TrimSpace(username))
cacheKey := fmt.Sprintf("algolia_creator_%s", username)
searchCreator := AlgoliaSearchCreator{}
cache, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(cache.([]uint8))
//log.Printf("CACHE: %d", len(cacheData))
//log.Printf("CACHEDATA: %#v", cacheData)
err = json.Unmarshal(cacheData, &searchCreator)
if err == nil {
return searchCreator, nil
}
}
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return searchCreator, errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("creators")
res, err := algoliaIndex.Search(username)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia creators (%s): %s", username, err)
return searchCreator, err
}
var newRecords []AlgoliaSearchCreator
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia creators: %s", err)
return searchCreator, err
}
//log.Printf("RECORDS: %d", len(newRecords))
foundUser := AlgoliaSearchCreator{}
for _, newRecord := range newRecords {
if strings.ToLower(newRecord.Username) == strings.ToLower(username) || newRecord.ObjectID == username || ArrayContainsLower(newRecord.Synonyms, username) {
foundUser = newRecord
break
}
}
// Handling search within a workflow, and in the future, within apps
if len(foundUser.ObjectID) == 0 {
if len(username) == 36 {
// Check workflows
algoliaIndex := algClient.InitIndex("workflows")
res, err := algoliaIndex.Search(username)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia creator workflow (%s): %s", username, err)
return searchCreator, err
}
var newRecords []AlgoliaSearchWorkflow
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia creator workflow: %s", err)
if len(newRecords) > 0 && len(newRecords[0].ObjectID) > 0 {
log.Printf("[INFO] Workflow search ID: %#v", newRecords[0].ObjectID)
} else {
return searchCreator, err
}
}
//log.Printf("[DEBUG] Got %d records for workflow sub", len(newRecords))
if len(newRecords) == 1 {
if len(newRecords[0].Creator) > 0 && username != newRecords[0].Creator {
foundCreator, err := HandleAlgoliaCreatorSearch(ctx, newRecords[0].Creator)
if err != nil {
return searchCreator, err
}
foundUser = foundCreator
} else {
return searchCreator, errors.New("User not found")
}
} else {
return searchCreator, errors.New("User not found")
}
} else {
return searchCreator, errors.New("User not found")
}
}
if project.CacheDb {
data, err := json.Marshal(foundUser)
if err != nil {
return foundUser, nil
}
err = SetCache(ctx, cacheKey, data, 30)
if err != nil {
log.Printf("[WARNING] Failed updating algolia username cache: %s", err)
}
}
return foundUser, nil
}
func HandleAlgoliaPartnerSearch(ctx context.Context, orgId string) (AlgoliaSearchPartner, error) {
cacheKey := fmt.Sprintf("algolia_partner_%s", orgId)
searchPartner := AlgoliaSearchPartner{}
cache, err := GetCache(ctx, cacheKey)
if err == nil {
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &searchPartner)
if err == nil {
return searchPartner, nil
}
}
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return AlgoliaSearchPartner{}, errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("partners")
res, err := algoliaIndex.Search(orgId)
if err != nil {
log.Printf("[WARNING] Failed searching Algolia partners: %s", err)
return AlgoliaSearchPartner{}, err
}
var newRecords []AlgoliaSearchPartner
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia partners: %s", err)
return AlgoliaSearchPartner{}, err
}
foundPartner := AlgoliaSearchPartner{}
for _, newRecord := range newRecords {
if newRecord.OrgId == orgId {
foundPartner = newRecord
break
}
}
if project.CacheDb {
data, err := json.Marshal(foundPartner)
if err != nil {
return foundPartner, nil
}
err = SetCache(ctx, cacheKey, data, 30)
if err != nil {
log.Printf("[WARNING] Failed updating algolia partner cache: %s", err)
}
}
return foundPartner, nil
}
func HandleAlgoliaCreatorUpload(ctx context.Context, user User, overwrite bool, isOrg bool) (string, error) {
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return "", errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("creators")
res, err := algoliaIndex.Search(user.Id)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia creators (%s): %s", user.Id, err)
return "", err
}
var newRecords []AlgoliaSearchCreator
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia creators: %s", err)
return "", err
}
//log.Printf("RECORDS: %d", len(newRecords))
for _, newRecord := range newRecords {
if newRecord.ObjectID == user.Id {
log.Printf("[INFO] Object %s already exists in Algolia", user.Id)
if overwrite {
break
} else {
return user.Id, errors.New("User ID already exists!")
}
}
}
timeNow := int64(time.Now().Unix())
records := []AlgoliaSearchCreator{
AlgoliaSearchCreator{
ObjectID: user.Id,
TimeEdited: timeNow,
Image: user.PublicProfile.GithubAvatar,
Username: user.PublicProfile.GithubUsername,
IsOrg: isOrg,
},
}
_, err = algoliaIndex.SaveObjects(records)
if err != nil {
log.Printf("[WARNING] Algolia Object put err: %s", err)
return "", err
}
log.Printf("[INFO] SUCCESSFULLY UPLOADED creator %s with ID %s TO ALGOLIA!", user.Username, user.Id)
return user.Id, nil
}
func HandleAlgoliaCreatorDeletion(ctx context.Context, userId string) error {
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("creators")
res, err := algoliaIndex.Search(userId)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia creators (%s): %s", userId, err)
return err
}
var newRecords []AlgoliaSearchCreator
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia creators: %s", err)
return err
}
//log.Printf("RECORDS: %d", len(newRecords))
foundItem := AlgoliaSearchCreator{}
for _, newRecord := range newRecords {
if newRecord.ObjectID == userId {
foundItem = newRecord
break
}
}
// Should delete it?
if len(foundItem.ObjectID) > 0 {
_, err = algoliaIndex.DeleteObject(foundItem.ObjectID)
if err != nil {
log.Printf("[WARNING] Algolia Creator delete problem: %s", err)
return err
}
log.Printf("[INFO] Successfully removed creator %s with ID %s FROM ALGOLIA!", foundItem.Username, userId)
}
return nil
}
// Usecase Algolia Upload
func HandleAlgoliaUsecaseUpload(ctx context.Context, usecase UsecaseInfo, overwrite bool) (string, error) {
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return "", errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("usecases")
res, err := algoliaIndex.Search(usecase.Id)
if err != nil {
log.Printf("[WARNING] Failed searching Algolia usecases: %s", err)
return "", err
}
var newRecords []AlgoliaSearchUsecase
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia partners: %s", err)
return "", err
}
//log.Printf("RECORDS: %d", len(newRecords))
for _, newRecord := range newRecords {
if newRecord.ObjectID == usecase.Id {
log.Printf("[INFO] Object %s already exists in Algolia", usecase.Id)
if overwrite {
break
} else {
return usecase.Id, errors.New("Usecase ID already exists!")
}
}
}
timeNow := int64(time.Now().Unix())
records := []AlgoliaSearchUsecase{
AlgoliaSearchUsecase{
ObjectID: usecase.Id,
PartnerName: usecase.CompanyInfo.Name,
PartnerId: usecase.CompanyInfo.Id,
Name: usecase.MainContent.Title,
Description: usecase.MainContent.Description,
Categories: usecase.MainContent.Categories,
SourceAppType: usecase.MainContent.SourceAppType,
DestinationAppType: usecase.MainContent.DestinationAppType,
PublicWorkflowID: usecase.MainContent.PublicWorkflowID,
TimeEdited: timeNow,
},
}
_, err = algoliaIndex.SaveObjects(records)
if err != nil {
log.Printf("[WARNING] Algolia Object put err: %s", err)
return "", err
}
log.Printf("[INFO] SUCCESSFULLY UPLOADED partner %s with ID %s TO ALGOLIA!", usecase.MainContent.Title, usecase.Id)
return usecase.Id, nil
}
// Usecase deletion
func HandleAlgoliaUsecaseDeletion(ctx context.Context, usecaseId string) error {
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("usecases")
res, err := algoliaIndex.Search(usecaseId)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia usecases (%s): %s", usecaseId, err)
return err
}
var newRecords []AlgoliaSearchUsecase
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia usecases: %s", err)
return err
}
//log.Printf("RECORDS: %d", len(newRecords))
foundItem := AlgoliaSearchUsecase{}
for _, newRecord := range newRecords {
if newRecord.ObjectID == usecaseId {
foundItem = newRecord
break
}
}
// Should delete it?
if len(foundItem.ObjectID) > 0 {
_, err = algoliaIndex.DeleteObject(foundItem.ObjectID)
if err != nil {
log.Printf("[WARNING] Algolia Usecase delete problem: %s", err)
return err
}
log.Printf("[INFO] Successfully removed usecase %s with ID %s FROM ALGOLIA!", foundItem.Name, usecaseId)
}
return nil
}
// Shitty temorary system
// Adding schedule to run over with another algorithm
// as well as this one, as to increase priority based on popularity:
// searches, clicks & conversions (CTR)
func GetWorkflowPriority(workflow Workflow) int {
prio := 0
if len(workflow.Tags) > 2 {
prio += 1
}
if len(workflow.Name) > 5 {
prio += 1
}
if len(workflow.Description) > 100 {
prio += 1
}
if len(workflow.WorkflowType) > 0 {
prio += 1
}
if len(workflow.UsecaseIds) > 0 {
prio += 3
}
if len(workflow.Comments) >= 2 {
prio += 2
}
return prio
}
func handleAlgoliaWorkflowUpdate(ctx context.Context, workflow Workflow) (string, error) {
log.Printf("[INFO] Should try to UPLOAD the Workflow to Algolia")
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) == 0 || len(algoliaSecret) == 0 {
log.Printf("[WARNING] ALGOLIA_CLIENT or ALGOLIA_SECRET not defined")
return "", errors.New("Algolia keys not defined")
}
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("workflows")
//res, err := algoliaIndex.Search("%s", api.ID)
res, err := algoliaIndex.Search(workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed searching Algolia (%s): %s", workflow.ID, err)
return "", err
}
var newRecords []AlgoliaSearchWorkflow
err = res.UnmarshalHits(&newRecords)
if err != nil {
log.Printf("[WARNING] Failed unmarshaling from Algolia workflow upload: %s", err)
return "", err
}
found := false
record := AlgoliaSearchWorkflow{}
for _, newRecord := range newRecords {
if newRecord.ObjectID == workflow.ID {
log.Printf("[INFO] Workflow Object %s already exists in Algolia", workflow.ID)
record = newRecord
found = true
break
}
}
if !found {
return "", errors.New(fmt.Sprintf("Couldn't find public workflow for ID %s", workflow.ID))
}
record.TimeEdited = int64(time.Now().Unix())
categories := []string{}
actions := []string{}
triggers := []string{}
actionRefs := []ActionReference{}
for _, action := range workflow.Actions {
if !ArrayContains(actions, action.AppName) {
// Using this API as the original is kinda stupid
foundApps, err := HandleAlgoliaAppSearchByUser(ctx, action.AppName)
if err == nil && len(foundApps) > 0 {
actionRefs = append(actionRefs, ActionReference{
Name: foundApps[0].Name,
Id: foundApps[0].ObjectID,
ImageUrl: foundApps[0].ImageUrl,
ActionName: []string{action.Name},
})
}
actions = append(actions, action.AppName)
} else {
for refIndex, ref := range actionRefs {
if ref.Name == action.AppName {
if !ArrayContains(ref.ActionName, action.Name) {
actionRefs[refIndex].ActionName = append(actionRefs[refIndex].ActionName, action.Name)
}
}
}
}
}
for _, trigger := range workflow.Triggers {
if !ArrayContains(triggers, trigger.TriggerType) {
triggers = append(triggers, trigger.TriggerType)
}
}
if workflow.WorkflowType != "" {
record.Type = workflow.WorkflowType
}
record.Name = workflow.Name
record.Description = workflow.Description
record.UsecaseIds = workflow.UsecaseIds
record.Triggers = triggers
record.Actions = actions
record.TriggerAmount = len(triggers)
record.ActionAmount = len(actions)
record.Tags = workflow.Tags
record.Categories = categories
record.ActionReferences = actionRefs
record.Priority = GetWorkflowPriority(workflow)
record.Validated = workflow.Validated
if len(workflow.Owner) > 0 {
record.Creator = workflow.Owner
}
records := []AlgoliaSearchWorkflow{
record,
}
//log.Printf("[WARNING] Returning before upload with data %#v", records)
//return records[0].ObjectID, nil
//return "", errors.New("Not prepared yet!")
_, err = algoliaIndex.SaveObjects(records)
if err != nil {
log.Printf("[WARNING] Algolia Object update err: %s", err)
return "", err
}
return workflow.ID, nil
}
// Returns an error if the users' org is over quota
func ValidateExecutionUsage(ctx context.Context, orgId string) (*Org, error) {
if len(orgId) == 0 {
return nil, errors.New("Org ID is empty")
}
org, err := GetOrg(ctx, orgId)
if err != nil {
return org, errors.New(fmt.Sprintf("Failed getting the organization %s: %s", orgId, err))
}
orgStats, err := GetOrgStatistics(ctx, orgId)
if err != nil {
log.Printf("[WARNING] Failed getting org statistics for %s (%s): %s", org.Name, org.Id, err)
return org, nil
}
if org.Billing.AppRunsHardLimit > 0 && orgStats.MonthlyAppExecutions > org.Billing.AppRunsHardLimit {
//log.Printf("[WARNING] Hard limit reached for org %s (%s) during exec start", org.Name, org.Id)
return org, errors.New(fmt.Sprintf("Org %s (%s) has exceeded the app runs hard limit (%d/%d). Your Parent organization can control this.", org.Name, org.Id, orgStats.MonthlyAppExecutions, org.Billing.AppRunsHardLimit))
}
validationOrg := org
validationOrgStats := orgStats
if len(org.CreatorOrg) > 0 {
validationOrg, err = GetOrg(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[WARNING] Failed getting creator org %s (%s): %s ", validationOrg.Name, validationOrg.Id, err)
//return org, errors.New(fmt.Sprintf("Failed getting the creator organization %s: %s", org.CreatorOrg, err))
return org, nil
}
validationOrgStats, err = GetOrgStatistics(ctx, org.CreatorOrg)
if err != nil {
log.Printf("[WARNING] Failed getting creator org statistics for %s (%s): %s ", validationOrg.Name, validationOrg.Id, err)
//return org, errors.New(fmt.Sprintf("Failed getting the creator organization statistics %s: %s", validationOrg.CreatorOrg, err))
return org, nil
}
}
// Fix Me: Add daily stats update script to append daily stats immdediately after day change and reset monthly stats on month change
lastMonthlyReset := validationOrgStats.LastMonthlyResetMonth
currentMonth := time.Now().UTC().Month()
if int(lastMonthlyReset) != int(currentMonth) {
validationOrgStats = handleDailyCacheUpdate(validationOrgStats)
err = SetOrgStatistics(ctx, *validationOrgStats, validationOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed setting org statistics for monthly reset for %s (%s): %s ", validationOrg.Name, validationOrg.Id, err)
}
}
totalAppExecutions := validationOrgStats.MonthlyAppExecutions + validationOrgStats.MonthlyChildAppExecutions
if validationOrg.Billing.InternalAppRunsHardLimit > 0 && totalAppExecutions > validationOrg.Billing.InternalAppRunsHardLimit {
return validationOrg, errors.New(fmt.Sprintf("Org %s (%s) has exceeded app runs hard limit (%d/%d) - Only Shuffle Support can control this metric.", validationOrg.Name, validationOrg.Id, totalAppExecutions, validationOrg.Billing.InternalAppRunsHardLimit))
}
// Allows partners and POV users to run workflows without limits
if validationOrg.LeadInfo.Internal || validationOrg.LeadInfo.ChannelPartner || validationOrg.LeadInfo.IntegrationPartner || validationOrg.LeadInfo.TechPartner || validationOrg.LeadInfo.DistributionPartner || validationOrg.LeadInfo.ServicePartner {
return validationOrg, nil
}
// If enterprise customer or pov then don't block them
if (validationOrg.LeadInfo.Customer || validationOrg.LeadInfo.POV) && validationOrg.SyncFeatures.AppExecutions.Limit >= 300000 {
return validationOrg, nil
}
if totalAppExecutions >= validationOrg.SyncFeatures.AppExecutions.Limit {
return validationOrg, errors.New(fmt.Sprintf("Org %s (%s) has exceeded the monthly app executions limit (%d/%d)", validationOrg.Name, validationOrg.Id, totalAppExecutions, validationOrg.SyncFeatures.AppExecutions.Limit))
}
if debug {
log.Printf("[INFO] Org %s (%s) has %d/%d app executions this month", validationOrg.Name, validationOrg.Id, totalAppExecutions, validationOrg.SyncFeatures.AppExecutions.Limit)
}
return validationOrg, nil
}
func RedirectUserRequest(w http.ResponseWriter, req *http.Request) {
if project.Environment == "cloud" && gceProject == "shuffler" {
log.Printf("[ERROR] Recursive RedirectRequest for %s", req.RequestURI)
w.WriteHeader(400)
w.Write([]byte(`{"success": false, "reason": "Recursive redirect request detected"}`))
return
}
proxyScheme := "https"
proxyHost := fmt.Sprintf("shuffler.io")
httpClient := &http.Client{
Timeout: 120 * time.Second,
}
body, err := ioutil.ReadAll(req.Body)
if err != nil {
log.Printf("[ERROR] Issue in SSR body proxy: %s", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//req.Body = ioutil.NopCloser(bytes.NewReader(body))
url := fmt.Sprintf("%s://%s%s", proxyScheme, proxyHost, req.RequestURI)
if debug {
log.Printf("[DEBUG] Request (%s) request URL: %s. More: %s", req.Method, url, req.URL.String())
}
proxyReq, err := http.NewRequest(req.Method, url, bytes.NewReader(body))
if err != nil {
log.Printf("[ERROR] Failed handling proxy request: %s", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// We may want to filter some headers, otherwise we could just use a shallow copy
proxyReq.Header = make(http.Header)
for h, val := range req.Header {
proxyReq.Header[h] = val
}
newresp, err := httpClient.Do(proxyReq)
if err != nil {
log.Printf("[ERROR] Issue in SSR newresp for %s - should retry: %s", url, err)
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
defer newresp.Body.Close()
urlbody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadGateway)
return
}
//log.Printf("RESP: %s", urlbody)
for key, value := range newresp.Header {
//log.Printf("%s %s", key, value)
for _, item := range value {
w.Header().Set(key, item)
}
}
w.WriteHeader(newresp.StatusCode)
w.Write(urlbody)
// Need to clear cache in case user gets updated in db
// with a new session and such. This only forces a new search,
// and shouldn't get them logged out
c, err := req.Cookie("session_token")
if err != nil {
c, err = req.Cookie("__session")
}
// FIXME: What is the point of this cookie checking?
if err == nil {
ctx := GetContext(req)
DeleteCache(ctx, fmt.Sprintf("session_%s", c.Value))
}
}
// Checks if a specific user should have "self" access to a creator user
// A creator user can be both a user and an org, so this got a bit tricky
func CheckCreatorSelfPermission(ctx context.Context, requestUser, creatorUser User, algoliaUser *AlgoliaSearchCreator) bool {
if project.Environment != "cloud" {
return false
}
if creatorUser.Id == requestUser.Id {
return true
} else {
for _, user := range algoliaUser.Synonyms {
if user == requestUser.Id {
return true
}
}
if algoliaUser.IsOrg {
log.Printf("[AUDIT] User %s (%s) is an org. Checking if the current user should have access.", algoliaUser.Username, algoliaUser.ObjectID)
// Get the org and check
org, err := GetOrgByCreatorId(ctx, algoliaUser.ObjectID)
if err != nil {
log.Printf("[WARNING] Couldn't find org for creator %s (%s): %s", algoliaUser.Username, algoliaUser.ObjectID, err)
return false
}
log.Printf("[AUDIT] Found org %s (%s) for creator %s (%s)", org.Name, org.Id, algoliaUser.Username, algoliaUser.ObjectID)
for _, user := range org.Users {
if user.Id == requestUser.Id {
if user.Role == "admin" {
return true
}
break
}
}
}
}
return false
}
// Uploads updates for a workflow to a specific file on git
func SetGitWorkflow(ctx context.Context, workflow Workflow, org *Org) error {
if workflow.BackupConfig.UploadRepo != "" || workflow.BackupConfig.UploadBranch != "" || workflow.BackupConfig.UploadUsername != "" || workflow.BackupConfig.UploadToken != "" {
//log.Printf("\n\n\n[DEBUG] Using workflow backup config for org %s (%s)\n\n\n", org.Name, org.Id)
org.Defaults.WorkflowUploadRepo = workflow.BackupConfig.UploadRepo
org.Defaults.WorkflowUploadBranch = workflow.BackupConfig.UploadBranch
org.Defaults.WorkflowUploadUsername = workflow.BackupConfig.UploadUsername
org.Defaults.WorkflowUploadToken = workflow.BackupConfig.UploadToken
// FIXME: Decrypt here
if workflow.BackupConfig.TokensEncrypted {
log.Printf("[DEBUG] Should realtime decrypt token for org %s (%s)", org.Name, org.Id)
org.Defaults.TokensEncrypted = true
} else {
org.Defaults.TokensEncrypted = false
}
}
if org.Defaults.TokensEncrypted == true {
log.Printf("[DEBUG] Decrypting token for org %s (%s)", org.Name, org.Id)
parsedKey := fmt.Sprintf("%s_upload_token", org.Id)
newValue, err := HandleKeyDecryption([]byte(org.Defaults.WorkflowUploadToken), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting token for org %s (%s): %s", org.Name, org.Id, err)
} else {
org.Defaults.WorkflowUploadToken = string(newValue)
}
parsedKey = fmt.Sprintf("%s_upload_username", org.Id)
newValue, err = HandleKeyDecryption([]byte(org.Defaults.WorkflowUploadUsername), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting username for org %s (%s): %s", org.Name, org.Id, err)
} else {
org.Defaults.WorkflowUploadUsername = string(newValue)
}
parsedKey = fmt.Sprintf("%s_upload_repo", org.Id)
newValue, err = HandleKeyDecryption([]byte(org.Defaults.WorkflowUploadRepo), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting repo for org %s (%s): %s", org.Name, org.Id, err)
} else {
org.Defaults.WorkflowUploadRepo = string(newValue)
}
parsedKey = fmt.Sprintf("%s_upload_branch", org.Id)
newValue, err = HandleKeyDecryption([]byte(org.Defaults.WorkflowUploadBranch), parsedKey)
if err != nil {
log.Printf("[ERROR] Failed decrypting branch for org %s (%s): %s", org.Name, org.Id, err)
} else {
org.Defaults.WorkflowUploadBranch = string(newValue)
}
log.Printf("[DEBUG] Decrypted token for org %s (%s): %s", org.Name, org.Id, newValue)
}
if len(org.Defaults.WorkflowUploadBranch) == 0 {
// Default to 'main' for Azure DevOps, 'master' for others
if strings.Contains(org.Defaults.WorkflowUploadRepo, "dev.azure.com") {
org.Defaults.WorkflowUploadBranch = "main"
} else {
org.Defaults.WorkflowUploadBranch = "master"
}
}
if org.Defaults.WorkflowUploadRepo == "" || org.Defaults.WorkflowUploadToken == "" {
//log.Printf("[DEBUG] Missing Repo/Token during Workflow backup upload for org %s (%s)", org.Name, org.Id)
//return errors.New("Missing repo or token")
return nil
}
org.Defaults.WorkflowUploadRepo = strings.TrimSpace(org.Defaults.WorkflowUploadRepo)
// Remove images from workflow before backup
workflow.Image = ""
for actionIndex, _ := range workflow.Actions {
workflow.Actions[actionIndex].LargeImage = ""
workflow.Actions[actionIndex].SmallImage = ""
}
for triggerIndex, _ := range workflow.Triggers {
workflow.Triggers[triggerIndex].LargeImage = ""
workflow.Triggers[triggerIndex].SmallImage = ""
}
// remove github backup info
workflow.BackupConfig = BackupConfig{}
// Use git to upload the workflow.
workflowData, err := json.MarshalIndent(workflow, "", " ")
if err != nil {
log.Printf("[ERROR] Failed marshalling workflow %s (%s) for git upload: %s", workflow.Name, workflow.ID, err)
return err
}
commitMessage := fmt.Sprintf("User '%s' updated workflow '%s' with status '%s' at %s", workflow.UpdatedBy, workflow.Name, workflow.Status, time.Now().Format("2006-01-02 15:04:05"))
repoURL := org.Defaults.WorkflowUploadRepo
repoURL = strings.TrimPrefix(repoURL, "https://")
repoURL = strings.TrimPrefix(repoURL, "http://")
var location string
var isAzureDevOps bool
if strings.Contains(repoURL, "dev.azure.com") {
isAzureDevOps = true
location = fmt.Sprintf("https://%s", repoURL)
log.Printf("[DEBUG] Detected Azure DevOps repository")
} else {
isAzureDevOps = false
// Only append .git if the URL contains github.com
if strings.Contains(repoURL, "github.com") && !strings.HasSuffix(repoURL, ".git") {
repoURL += ".git"
}
urlEncodedPassword := url.QueryEscape(org.Defaults.WorkflowUploadToken)
location = fmt.Sprintf("https://%s:%s@%s", org.Defaults.WorkflowUploadUsername, urlEncodedPassword, repoURL)
}
maskedRepo := location
if !isAzureDevOps {
maskedRepo = strings.ReplaceAll(location, org.Defaults.WorkflowUploadToken, "****")
if urlEncodedPassword := url.QueryEscape(org.Defaults.WorkflowUploadToken); urlEncodedPassword != org.Defaults.WorkflowUploadToken {
maskedRepo = strings.ReplaceAll(maskedRepo, urlEncodedPassword, "****")
}
}
log.Printf("[DEBUG] Uploading workflow %s to repo: %s", workflow.ID, maskedRepo)
fs := memfs.New()
if len(workflow.Status) == 0 {
workflow.Status = "test"
}
//filePath := fmt.Sprintf("/%s/%s.json", workflow.Status, workflow.ID)
filePath := fmt.Sprintf("%s/%s/%s_%s.json", workflow.ExecutingOrg.Id, workflow.Status, strings.ReplaceAll(workflow.Name, " ", "-"), workflow.ID)
cloneOptions := &git.CloneOptions{
URL: location,
}
// For Azure DevOps, add additional auth configuration and capability handling
if isAzureDevOps {
transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}
cloneOptions.Auth = &gitHttp.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername, // Use the username for Azure DevOps
Password: org.Defaults.WorkflowUploadToken,
}
}
repo, err := git.Clone(memory.NewStorage(), fs, cloneOptions)
if err != nil {
errMsg := err.Error()
if isAzureDevOps {
log.Printf("[ERROR] Azure DevOps clone failed. Check: 1) PAT has Code(R/W) permissions, 2) Branch '%s' exists, 3) Repo URL is correct", org.Defaults.WorkflowUploadBranch)
}
log.Printf("[ERROR] Error cloning repo '%s': %s", maskedRepo, strings.ReplaceAll(errMsg, org.Defaults.WorkflowUploadToken, "****"))
return err
}
w, err := repo.Worktree()
if err != nil {
log.Printf("[ERROR] Error getting worktree for repo '%s': %s", maskedRepo, err)
return err
}
// Write the byte blob to the in-memory file system
file, err := fs.Create(filePath)
if err != nil {
log.Printf("[ERROR] Creating file in repo '%s': %v", maskedRepo, err)
return err
}
defer file.Close()
if _, err = io.Copy(file, bytes.NewReader(workflowData)); err != nil {
log.Printf("[ERROR] Writing data to file: %v", err)
return err
}
if _, err = w.Add(filePath); err != nil {
log.Printf("[ERROR] Adding file to staging area: %s", err)
return err
}
// Check if there are any changes to commit
status, err := w.Status()
if err != nil {
log.Printf("[ERROR] Getting working tree status: %v", err)
return err
}
hasChanges := false
for _, fileStatus := range status {
if fileStatus.Staging != git.Unmodified {
hasChanges = true
break
}
}
if !hasChanges {
log.Printf("[INFO] No changes detected for workflow %s (%s). File content is identical to existing version.", workflow.Name, workflow.ID)
return nil
}
authorName := org.Defaults.WorkflowUploadUsername
if authorName == "" {
if isAzureDevOps {
authorName = "Workflow Automation"
} else {
authorName = "Shuffle User"
}
}
_, err = w.Commit(commitMessage, &git.CommitOptions{
Author: &object.Signature{
Name: authorName,
Email: "",
When: time.Now(),
},
})
if err != nil {
log.Printf("[ERROR] Committing changes: %v", err)
return err
}
//log.Printf("[DEBUG] Commit Hash: %s", commit)
// Push the changes to a remote repository (replace URL with your repository URL)
// fmt.Sprintf("refs/heads/%s:refs/heads/%s", org.Defaults.WorkflowUploadBranch, org.Defaults.WorkflowUploadBranch)},
ref := fmt.Sprintf("refs/heads/%s:refs/heads/%s", org.Defaults.WorkflowUploadBranch, org.Defaults.WorkflowUploadBranch)
pushOptions := &git.PushOptions{
RemoteName: "origin",
RefSpecs: []config.RefSpec{config.RefSpec(ref)},
}
// Set authentication for push operations for both Azure DevOps and GitHub
pushOptions.Auth = &gitHttp.BasicAuth{
Username: org.Defaults.WorkflowUploadUsername,
Password: org.Defaults.WorkflowUploadToken,
}
err = repo.Push(pushOptions)
if err != nil {
log.Printf("[ERROR] Git push failed for repo '%s': %v", maskedRepo, err)
return err
}
log.Printf("[DEBUG] Workflow successfully uploaded to '%s'!", maskedRepo)
return nil
}
// Creates osfs from folderpath with a basepath as directory base
func CreateFs(basepath, pathname string) (billy.Filesystem, error) {
log.Printf("[INFO] MemFS base: %s, pathname: %s", basepath, pathname)
fs := memfs.New()
err := filepath.Walk(pathname,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if strings.Contains(path, ".git") {
return nil
}
// Fix the inner path here
newpath := strings.ReplaceAll(path, pathname, "")
fullpath := fmt.Sprintf("%s%s", basepath, newpath)
switch mode := info.Mode(); {
case mode.IsDir():
err = fs.MkdirAll(fullpath, 0644)
if err != nil {
log.Printf("Failed making folder: %s", err)
}
case mode.IsRegular():
srcData, err := ioutil.ReadFile(path)
if err != nil {
log.Printf("Src error: %s", err)
return err
}
dst, err := fs.Create(fullpath)
if err != nil {
log.Printf("Dst error: %s", err)
return err
}
_, err = dst.Write(srcData)
if err != nil {
log.Printf("Dst write error: %s", err)
return err
}
}
return nil
})
return fs, err
}
// Also deactivates. It's a toggle for off and on.
func ActivateWorkflowApp(resp http.ResponseWriter, request *http.Request) {
cors := HandleCors(resp, request)
if cors {
return
}
user, err := HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in get active apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role == "org-reader" {
log.Printf("[WARNING] Org-reader doesn't have access to activate workflow app (shared): %s (%s)", user.Username, user.Id)
resp.WriteHeader(403)
resp.Write([]byte(`{"success": false, "reason": "Read only user"}`))
return
}
ctx := GetContext(request)
location := strings.Split(request.URL.String(), "/")
var appId string
activate := true
shouldDistributeToLocation := false
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
appId = location[4]
if strings.ToLower(location[5]) == "deactivate" {
activate = false
}
if strings.ToLower(location[5]) == "distribute" {
shouldDistributeToLocation = true
}
}
// If onprem, it should autobuild the container(s) from here
// FIXME: The problem with redirect:
// 1. You are in EU
// 2. You activate an app for a suborg, which has to be saved to ActivatedApps in the org in EU so that it loads properly
// 3. The app itself is in EU - NOT in UK, but the org has to be updated in UK -> EU propagation
// 4. In this case, it would overwrite the current change as well
// Additional: Superfluous response(s)
if project.Environment == "cloud" && gceProject != "shuffler" {
go LoadAppConfigFromMain(appId, false)
}
// This is a special case where auth was handled in another region, and activation is done anyway
if project.Environment == "cloud" && gceProject == "shuffler" && request.URL.Query().Get("propagation") == os.Getenv("SHUFFLE_PROPAGATE_TOKEN") && len(os.Getenv("SHUFFLE_PROPAGATE_TOKEN")) > 0 {
log.Printf("[AUDIT] User %s (%s) is activating app %s for org %s (%s) with their org auth token (distributed)", user.Username, user.Id, appId, user.ActiveOrg.Name, user.ActiveOrg.Id)
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s (%s) for app activation: %s (prop)", user.ActiveOrg.Name, user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org"}`))
return
}
added := false
if activate {
if !ArrayContains(org.ActiveApps, appId) {
org.ActiveApps = append(org.ActiveApps, appId)
added = true
} else if ArrayContains(org.ActiveApps, appId) {
// If the app is already in the org, we don't need to add it again
log.Printf("[DEBUG] App %s already exists in org %s (%s). Not adding again (prop)", appId, user.ActiveOrg.Name, user.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "App already exists in org"}`))
return
}
} else {
// Remove from the array
newActiveApps := []string{}
for _, activeApp := range org.ActiveApps {
if activeApp == appId {
continue
}
newActiveApps = append(newActiveApps, activeApp)
}
org.ActiveApps = newActiveApps
added = true
}
if added {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[ERROR] Failed setting org %s (%s) after activating app %s: %s (propagate!)", user.ActiveOrg.Name, user.ActiveOrg.Id, appId, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": true, "reason": "Failed setting org after activating app"}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "App activated"}`))
return
} else {
resp.WriteHeader(201)
resp.Write([]byte(`{"success": true, "reason": "App already handled"}`))
return
}
}
app, err := GetApp(ctx, appId, user, false)
if err != nil {
appName := request.URL.Query().Get("app_name")
appVersion := request.URL.Query().Get("app_version")
if len(appName) > 0 && len(appVersion) > 0 {
apps, err := FindWorkflowAppByName(ctx, appName)
//log.Printf("[INFO] Found %d apps for %s", len(apps), appName)
if err != nil || len(apps) == 0 {
log.Printf("[WARNING] Error getting app from name '%s' (app config): %s", appName, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
return
}
selectedApp := WorkflowApp{}
for _, app := range apps {
if !app.Sharing && !app.Public {
continue
}
if app.Name == appName {
selectedApp = app
}
if app.Name == appName && app.AppVersion == appVersion {
selectedApp = app
}
}
app = &selectedApp
} else {
log.Printf("[WARNING] Error getting app with ID %s (app config): %s", appId, err)
// Automatic propagation to cloud regions
if project.Environment == "cloud" && gceProject != "shuffler" {
app, err := HandleAlgoliaAppSearch(ctx, appId)
if err == nil {
// this means that the app exists. so, let's
// ask our propagator to proagate it further.
log.Printf("[INFO] Found apps %s - %s in algolia", app.Name, app.ObjectID)
if app.ObjectID != appId {
log.Printf("[WARNING] App %s doesn't exist in algolia", appId)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
return
}
// i can in theory, run this without using goroutines
// and then recursively call the same function. but that
// would make this request way too long.
go func() {
err = propagateApp(appId, false)
if err != nil {
log.Printf("[WARNING] Error propagating app %s - %s: %s", app.Name, app.ObjectID, err)
} else {
log.Printf("[INFO] Propagated app %s - %s. Sending request again!", app.Name, app.ObjectID)
}
}()
resp.WriteHeader(202)
resp.Write([]byte(`{"success": false, "reason": "Taking care of some magic. Please try activation again in a few seconds!"}`))
return
} else {
log.Printf("[WARNING] Error getting app %s (algolia): %s", appName, err)
}
} else if project.Environment == "cloud" && gceProject == "shuffler" {
// Automatic deletion in the main region if the app doesn't exist
if len(app.ID) == 0 {
log.Printf("[INFO] Auto-Removing app %s from Algolia as it doesn't exist with the same ID anymore. Request source: %s (%s) in org %s (%s)", appId, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
algoliaClient := os.Getenv("ALGOLIA_CLIENT")
algoliaSecret := os.Getenv("ALGOLIA_SECRET")
if len(algoliaClient) > 0 && len(algoliaSecret) > 0 {
algClient := search.NewClient(algoliaClient, algoliaSecret)
algoliaIndex := algClient.InitIndex("appsearch")
_, err = algoliaIndex.DeleteObject(appId)
if err != nil {
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed removing the app from Algolia"}`))
return
}
}
}
}
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "App doesn't exist"}`))
return
}
}
if activate == false && app.ReferenceOrg == user.ActiveOrg.Id {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "Can't remove app from current org as it is the owner org."}`))
return
}
distributingApp := false
if !app.Public && app.ReferenceOrg != user.ActiveOrg.Id && user.ActiveOrg.Role == "admin" {
if app.Owner == user.Id {
log.Printf("[INFO] App %s (%s) is owned by the user %s (%s). Distributing it to the org %s (%s)", app.Name, app.ID, user.Username, user.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
distributingApp = true
} else {
// check if the app belongs to parent org
org, err := GetOrg(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting org %s (%s): %s", user.ActiveOrg.Name, user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting org"}`))
return
}
if org.CreatorOrg == app.ReferenceOrg {
log.Printf("[INFO] App %s (%s) is owned by the parent org %s (%s). Distributing it to the suborg %s (%s)", app.Name, app.ID, org.Name, org.Id, user.ActiveOrg.Name, user.ActiveOrg.Id)
distributingApp = true
}
}
}
org := &Org{}
added := false
if app.Sharing || app.Public || !activate || distributingApp {
org, err = GetOrg(ctx, user.ActiveOrg.Id)
if err == nil {
if len(org.ActiveApps) > 150 {
// No reason for it to be this big. Arbitrarily reducing.
same := []string{}
samecnt := 0
for _, activeApp := range org.ActiveApps {
if ArrayContains(same, activeApp) {
samecnt += 1
continue
}
same = append(same, activeApp)
}
added = true
//log.Printf("Same: %d, total uniq: %d", samecnt, len(same))
org.ActiveApps = org.ActiveApps[len(org.ActiveApps)-100 : len(org.ActiveApps)-1]
}
if activate {
if !ArrayContains(org.ActiveApps, app.ID) {
org.ActiveApps = append(org.ActiveApps, app.ID)
added = true
} else if ArrayContains(org.ActiveApps, app.ID) && !app.Public {
// If the app is already in the org, we don't need to add it again
log.Printf("[DEBUG] App %s (%s) already exists in org %s (%s). Not adding again.", app.Name, app.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "App already exists in org"}`))
return
}
} else {
// Remove from the array
newActiveApps := []string{}
for _, activeApp := range org.ActiveApps {
if activeApp == app.ID {
continue
}
newActiveApps = append(newActiveApps, activeApp)
}
org.ActiveApps = newActiveApps
added = true
}
if added {
err = SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[WARNING] Failed setting org when autoadding apps on save: %s", err)
} else {
addRemove := "Added"
if !activate {
addRemove = "Removed"
}
log.Printf("[INFO] %s public app %s (%s) to/from org %s (%s). Activated apps: %d", addRemove, app.Name, app.ID, user.ActiveOrg.Name, user.ActiveOrg.Id, len(org.ActiveApps))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
if project.Environment == "cloud" && gceProject != "shuffler" {
// propagate org.ActiveApps to the main region
go func() {
// wait for a second before propagating again
log.Printf("[INFO] Propagating org %s after sleeping for a second!", user.ActiveOrg.Id)
time.Sleep(1 * time.Second)
err = propagateOrg(*org, true)
if err != nil {
log.Printf("[WARNING] Error propagating org %s: %s", user.ActiveOrg.Id, err)
}
}()
}
}
}
}
} else {
// Check if the user in the current context should have access to it or not
if user.Id == app.Owner || user.ActiveOrg.Id == app.ReferenceOrg || ArrayContains(app.Contributors, user.Id) {
log.Printf("[AUDIT] User %s (%s) is activating app %s (%s) for org %s (%s)", user.Username, user.Id, app.Name, app.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
} else if project.Environment == "cloud" && user.Verified == true && user.Active == true && user.SupportAccess == true && strings.HasSuffix(user.Username, "@shuffler.io") {
log.Printf("[AUDIT] User %s (%s) is activating app %s (%s) for org %s (%s) as a support user", user.Username, user.Id, app.Name, app.ID, user.ActiveOrg.Name, user.ActiveOrg.Id)
} else {
foundOrg := &Org{}
for _, org := range user.Orgs {
if org != app.ReferenceOrg {
continue
}
foundOrg, err = GetOrg(ctx, org)
if err != nil {
log.Printf("[ERROR] Failed getting org %s: %s", org, err)
}
break
}
allowed := false
if foundOrg.Id == app.ReferenceOrg {
for _, foundUser := range foundOrg.Users {
if foundUser.Id == user.Id && foundUser.Role != "org-reader" {
allowed = true
break
}
}
}
if !allowed {
log.Printf("[WARNING] User is trying to activate %s which is NOT a public app", app.Name)
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false}`))
return
}
}
}
if shouldDistributeToLocation {
// Distribute to runtime Locations
allEnvironments, err := GetEnvironments(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed getting environments for org %s: %s", user.ActiveOrg.Id, err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed getting environments"}`))
return
}
relevantEnvironments := []Environment{}
for _, env := range allEnvironments {
if strings.ToLower(env.Type) == "cloud" {
continue
}
if env.Archived {
continue
}
relevantEnvironments = append(relevantEnvironments, env)
}
if len(relevantEnvironments) == 0 {
resp.WriteHeader(400)
resp.Write([]byte(`{"success": false, "reason": "No relevant environments"}`))
return
}
appName := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(app.Name, " ", "-")), app.AppVersion)
if project.Environment == "cloud" {
if app.Public == true {
} else {
appName = fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(app.Name, " ", "-")), app.ID)
}
}
for _, env := range relevantEnvironments {
//log.Printf("[INFO] Distributing app %s to environment %s", app.Name, env.Name)
request := ExecutionRequest{
Type: "DOCKER_IMAGE_DOWNLOAD",
ExecutionId: uuid.NewV4().String(),
ExecutionArgument: fmt.Sprintf("frikky/shuffle:%s", appName),
Priority: 11,
}
parsedId := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(env.Name, " ", "-"), "_", "-")), env.OrgId)
err = SetWorkflowQueue(ctx, request, parsedId)
if err != nil {
log.Printf("[ERROR] Failed setting workflow queue for env: %s", err)
continue
}
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "reason": "Re-download request sent to all relevant environments"}`))
return
}
if activate {
log.Printf("[DEBUG] App %s (%s) activated for org %s by user %s (%s). Active apps: %d. Already existed: %t", app.Name, app.ID, user.ActiveOrg.Id, user.Username, user.Id, len(org.ActiveApps), !added)
} else {
log.Printf("[DEBUG] App %s (%s) deactivated for org %s by user %s (%s). Active apps: %d. Already existed: %t", app.Name, app.ID, user.ActiveOrg.Id, user.Username, user.Id, len(org.ActiveApps), !added)
}
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.ActiveOrg.Id))
DeleteCache(ctx, fmt.Sprintf("apps_%s", user.Id))
DeleteCache(ctx, "all_apps")
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-100"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-500"))
DeleteCache(ctx, fmt.Sprintf("workflowapps-sorted-1000"))
reason := "App Activated"
if !activate {
reason = "App Deactivated"
}
// Happens down here as we want to make sure the auth is done
if project.Environment == "cloud" && gceProject != "shuffler" {
if len(os.Getenv("SHUFFLE_PROPAGATE_TOKEN")) > 0 {
newQuery := fmt.Sprintf("propagation=%s", os.Getenv("SHUFFLE_PROPAGATE_TOKEN"))
requestUrl := request.URL.String()
if !strings.Contains(requestUrl, "?") {
requestUrl = fmt.Sprintf("%s?%s", requestUrl, newQuery)
} else {
requestUrl = fmt.Sprintf("%s&%s", requestUrl, newQuery)
}
parsedUrl, err := url.Parse(requestUrl)
if err != nil {
log.Printf("[ERROR] Failed parsing request URL for redirect: %s", err)
} else {
request.URL = parsedUrl
}
}
go RedirectUserRequest(resp, request)
return
// Just to ensure org propagation across regions occurs in time
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true,"reason": "%s"}`, reason)))
}
// For replicating HTTP request from schedule user
func HandleSuborgScheduleRun(request *http.Request, workflow *Workflow) {
ctx := context.Background()
if len(workflow.SuborgDistribution) == 0 {
log.Printf("[WARNING] No suborgs to run for workflow %s", workflow.ID)
return
}
// Finding first one.
originalTriggerId := ""
for _, trigger := range workflow.Triggers {
if trigger.TriggerType == "SCHEDULE" {
originalTriggerId = trigger.ID
break
}
}
if len(originalTriggerId) == 0 {
return
}
// 1. Get child workflows of workflow
// 2. Map to the right ones
childWorkflows, err := ListChildWorkflows(ctx, workflow.ID)
if err != nil {
log.Printf("[ERROR] Failed getting child workflows for parent workflow %s: %s", workflow.ID, err)
return
}
client := http.Client{}
for _, childWorkflow := range childWorkflows {
if childWorkflow.ID == workflow.ID {
continue
}
if childWorkflow.OrgId == workflow.OrgId {
continue
}
// Check if the OrgId is still in the workflow.Sub
found := false
for _, suborg := range workflow.SuborgDistribution {
if childWorkflow.OrgId == suborg {
found = true
break
}
}
if !found {
continue
}
// Ensuring the trigger still exists in the child
found = false
for _, trigger := range childWorkflow.Triggers {
if trigger.ReplacementForTrigger == originalTriggerId {
found = true
break
}
}
if !found {
continue
}
log.Printf("[DEBUG] Should be running %s schedule suborg workflows", childWorkflow.ID)
go func(client http.Client, request *http.Request, childWorkflow Workflow) {
baseurl := "https://shuffler.io"
if os.Getenv("BASE_URL") != "" {
baseurl = os.Getenv("BASE_URL")
}
if os.Getenv("SHUFFLE_CLOUDRUN_URL") != "" {
baseurl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("[ERROR] Failed reading body from schedule request: %s", err)
return
}
request.Body = io.NopCloser(bytes.NewBuffer(body))
formattedUrl := fmt.Sprintf("%s/api/v1/workflows/%s/run", baseurl, childWorkflow.ID)
req, err := http.NewRequest(
"POST",
formattedUrl,
bytes.NewBuffer(body),
)
if err != nil {
log.Printf("[WARNING] Failed mapping child workflow schedule: %s", err)
return
}
for key, value := range request.Header {
req.Header.Set(key, value[0])
}
newresp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR] Failed running child workflow schedule: %s", err)
return
}
defer newresp.Body.Close()
if newresp.StatusCode == 200 {
log.Printf("[DEBUG] Started suborg workflow from schedule. Parent: %s. Child: %s", childWorkflow.ParentWorkflowId, childWorkflow.ID)
} else {
respBody, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed to read body from failed newresp")
return
}
log.Printf("[ERROR] Failed to start suborg workflow from schedule with status %d. Parent: %s. Child: %s. Raw Body: %s", newresp.StatusCode, childWorkflow.ParentWorkflowId, childWorkflow.ID, string(respBody))
}
}(client, request, childWorkflow)
}
}
// This is JUST for Singul actions with AI agents.
// As AI Agents can have multiple types of runs, this could change every time.
func RunAgentDecisionSingulActionHandler(execution WorkflowExecution, decision AgentDecision) ([]byte, string, string, []string, string, error) {
debugUrl := ""
log.Printf("[INFO][%s] Running agent decision action '%s' with app '%s'. This is ran with Singul.", execution.ExecutionId, decision.Action, decision.Tool)
// Check if running in test mode
if os.Getenv("AGENT_TEST_MODE") == "true" {
log.Printf("[DEBUG][%s] AGENT_TEST_MODE enabled - using mock tool execution", execution.ExecutionId)
// Call mock handler
body, debugUrl, appName, err := RunAgentDecisionMockHandler(execution, decision)
return body, debugUrl, appName, []string{}, "", err
}
baseUrl := "https://shuffler.io"
if os.Getenv("BASE_URL") != "" {
baseUrl = os.Getenv("BASE_URL")
}
if os.Getenv("SHUFFLE_CLOUDRUN_URL") != "" {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
requestUrl := fmt.Sprintf("%s/api/v1/apps/categories/run?authorization=%s&execution_id=%s", baseUrl, execution.Authorization, execution.ExecutionId)
// Change timeout to be 300 seconds (just in case)
// Allows for reruns and self-correcting
client := GetExternalClient(requestUrl)
client.Timeout = 300 * time.Second
newFields := []schemaless.Valuereplace{}
for _, field := range decision.Fields {
newFields = append(newFields, schemaless.Valuereplace{
Key: field.Key,
Value: field.Value,
Answer: field.Answer,
})
}
parsedFields := schemaless.TranslateBadFieldFormats(newFields)
// Check if this is a GET request and strip the body field if present
// GET requests should not have a body and can cause 400 errors
methodValue := ""
for _, field := range parsedFields {
if strings.ToLower(field.Key) == "method" {
methodValue = strings.ToUpper(field.Value)
break
}
}
if (strings.ToLower(decision.Action) == "custom_action" || strings.ToLower(decision.Action) == "api") && strings.ToLower(decision.Tool) != "http" {
var urlValue string
newFields := make([]schemaless.Valuereplace, 0, len(parsedFields))
// Extract url field and keep all non-url fields
for _, field := range parsedFields {
if strings.ToLower(field.Key) == "url" && field.Value != "" {
urlValue = strings.TrimSpace(field.Value)
continue
}
newFields = append(newFields, field)
}
if urlValue != "" {
var path string
if strings.HasPrefix(urlValue, "http://") || strings.HasPrefix(urlValue, "https://") {
if u, err := url.Parse(urlValue); err == nil {
path = u.Path
if u.RawQuery != "" {
path = path + "?" + u.RawQuery
}
}
} else {
path = "/" + strings.TrimLeft(urlValue, "/")
}
if path != "" {
newFields = append(newFields, schemaless.Valuereplace{
Key: "path",
Value: path,
})
if debug {
log.Printf("[DEBUG][%s] Converted url to path for %s: path='%s' (auth base URL will be used)", execution.ExecutionId, decision.Tool, path)
}
}
}
parsedFields = newFields
}
oldFields := []Valuereplace{}
for _, field := range parsedFields {
// Skip body field for GET requests
if methodValue == "GET" && strings.ToLower(field.Key) == "body" {
log.Printf("[INFO][%s] Stripping 'body' field from GET request to %s", execution.ExecutionId, decision.Tool)
continue
}
oldFields = append(oldFields, Valuereplace{
Key: field.Key,
Value: field.Value,
Answer: field.Answer,
})
}
parsedAction := CategoryAction{
AppName: decision.Tool,
Label: decision.Action,
Query: decision.Reason, // Add the reason field for LLM context
Fields: oldFields,
}
if strings.ToLower(decision.Action) == "api" {
parsedAction.Action = "custom_action"
}
marshalledAction, err := json.Marshal(parsedAction)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed marshalling action in agent decision: %s", execution.ExecutionId, err)
return []byte{}, debugUrl, decision.Tool, []string{}, "", err
}
req, err := http.NewRequest(
"POST",
requestUrl,
bytes.NewBuffer(marshalledAction),
)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed creating request for agent decision: %s", execution.ExecutionId, err)
return []byte{}, debugUrl, decision.Tool, []string{}, "", err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed running agent decision (1). Timeout: %d: %s", execution.ExecutionId, client.Timeout, err)
return []byte{}, debugUrl, decision.Tool, []string{}, "", err
}
appname := decision.Tool
appId := ""
_ = appId
for key, value := range resp.Header {
//if debug {
// log.Printf("\n\n\n\n[DEBUG][%s] HEADER: key: %s, value: %s\n\n\n\n", execution.ExecutionId, key, value)
//}
if key == "X-Appname" && len(value) > 0 {
appname = value[0]
continue
}
if key == "X-Appid" && len(value) > 0 {
appId = value[0]
continue
}
if key != "X-Debug-Url" {
continue
}
/*
if !strings.HasPrefix(key, "X-") {
continue
}
// Don't care about raw response
if key == "X-Raw-Response-Url" || key == "X-Apprun-Url" {
continue
}
*/
foundValue := ""
for _, val := range value {
if len(val) > 0 {
foundValue = val
break
}
}
debugUrl = foundValue
/*
returnHeaders = append(returnHeaders, Valuereplace{
Key: key,
Value: foundValue,
})
*/
}
originalBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed reading body from agent decision: %s", execution.ExecutionId, err)
return []byte{}, debugUrl, appname, []string{}, "", err
}
body := originalBody
defer resp.Body.Close()
// Try to map it into SchemalessOutput and grab "RawResponse"
outputMapped := SchemalessOutput{}
err = json.Unmarshal(body, &outputMapped)
if err != nil {
log.Printf("[ERROR] AI Agent: Failed unmarshalling agent decision response: %s", err)
return body, debugUrl, appname, []string{}, "", nil
}
if val, ok := outputMapped.RawResponse.(string); ok {
parsedVal, err := base64.StdEncoding.DecodeString(val)
if err == nil && (strings.HasPrefix(string(parsedVal), "{") && strings.HasSuffix(string(parsedVal), "}")) || (strings.HasPrefix(string(parsedVal), "[") && strings.HasSuffix(string(parsedVal), "]")) {
body = parsedVal
} else {
body = []byte(val)
}
} else if val, ok := outputMapped.RawResponse.([]byte); ok {
body = val
} else if val, ok := outputMapped.RawResponse.(map[string]interface{}); ok {
marshalledRawResp, err := json.MarshalIndent(val, "", " ")
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed marshalling agent decision response: %s", execution.ExecutionId, err)
} else {
body = marshalledRawResp
}
} else if outputMapped.RawResponse == nil {
// Do nothing
} else {
log.Printf("[ERROR][%s] AI Agent: FAILED MAPPING RAW RESP INTERfACE. TYPE: %T\n\n\n", execution.ExecutionId, outputMapped.RawResponse)
}
if resp.StatusCode != 200 {
if debug {
log.Printf("[ERROR][%s] AI Agent: Failed running agent decision with status %d: %s", execution.ExecutionId, resp.StatusCode, string(body))
} else {
log.Printf("[ERROR][%s] AI Agent: Failed running agent decision with status %d. Body: %d", execution.ExecutionId, resp.StatusCode, len(body))
}
return body, debugUrl, appname, []string{}, "", errors.New(fmt.Sprintf("Failed running agent decision (2). Status code %d", resp.StatusCode))
}
if outputMapped.Success == false {
return originalBody, debugUrl, appname, []string{}, "", errors.New("Failed running agent decision (3). Success false for Singul action")
}
/*
agentOutput.Decisions[decisionIndex].RunDetails.RawResponse = string(rawResponse)
agentOutput.Decisions[decisionIndex].RunDetails.DebugUrl = debugUrl
if err != nil {
log.Printf("[ERROR] Failed to run agent decision %#v: %s", decision, err)
agentOutput.Decisions[decisionIndex].RunDetails.Status = "FAILED"
resultMapping.Status = "FAILURE"
resultMapping.CompletedAt = time.Now().Unix()
agentOutput.CompletedAt = time.Now().Unix()
} else {
agentOutput.Decisions[decisionIndex].RunDetails.Status = "RUNNING"
}
*/
return body, debugUrl, appname, outputMapped.CategoryLabels, outputMapped.ActionName, nil
}
// Runs an Agent Decision -> returns the result from it
// FIXME: Handle types: https://www.figma.com/board/V6Kg7KxbmuhIUyTImb20t1/Shuffle-AI-Agent-system?node-id=0-1&p=f&t=yIGaSXQYsYReR8cI-0
// This function should handle:
// 1. Running the decided action (user input, Singul, Workflow, Other Agent, Custom HTTP function)
// 2. Taking the result and sending (?) it back
// 3. Ensuring cache for an action is kept up to date
func RunAgentDecisionAction(execution WorkflowExecution, agentOutput AgentOutput, decision AgentDecision) {
defer func() {
if r := recover(); r != nil {
log.Printf("[ERROR] AI_AGENT_PANIC: execution_id=%s decision_id=%s panic=%v", execution.ExecutionId, decision.RunDetails.Id, r)
// Mark decision as failed so agent doesn't get stuck
decision.RunDetails.Status = "FAILURE"
decision.RunDetails.CompletedAt = time.Now().UnixMilli()
decision.RunDetails.RawResponse = fmt.Sprintf("PANIC: %v", r)
}
}()
// Check if it's already ran or not
ctx := context.Background()
decisionId := fmt.Sprintf("agent-%s-%s", execution.ExecutionId, decision.RunDetails.Id)
cache, err := GetCache(ctx, decisionId)
if err == nil {
foundDecision := AgentDecision{}
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &foundDecision)
if err != nil {
log.Printf("[WARNING][%s] Failed agent decision unmarshal (not critical): %s", execution.ExecutionId, err)
}
if foundDecision.RunDetails.StartedAt > 0 {
log.Printf("[DEBUG][%s] Decision %s already has status '%s'. Returning as it's already started..", execution.ExecutionId, decision.RunDetails.Id, foundDecision.RunDetails.Status)
return
}
}
// Set it to this at the start
if decision.RunDetails.StartedAt <= 0 {
decision.RunDetails.StartedAt = time.Now().UnixMilli()
}
decision.RunDetails.Status = "RUNNING"
marshalledDecision, err := json.Marshal(decision)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed marshalling decision %s", execution.ExecutionId, decision.RunDetails.Id)
}
go SetCache(ctx, decisionId, marshalledDecision, 300)
if decision.Action == "user_input" || decision.Action == "answer" || decision.Action == "ask" || decision.Action == "question" || decision.Action == "finish" || decision.Category == "standalone" {
} else {
// Singul handler
rawResponse, debugUrl, appname, categoryLabels, actionName, err := RunAgentDecisionSingulActionHandler(execution, decision)
if len(appname) > 0 {
decision.Tool = appname
}
decision.RunDetails.RawResponse = string(rawResponse)
decision.RunDetails.DebugUrl = debugUrl
decision.RunDetails.CategoryLabels = categoryLabels
decision.RunDetails.ActionName = actionName
log.Printf("RawResp: %s", string(rawResponse))
if err != nil {
if debug {
log.Printf("[ERROR][%s] AI Agent: Failed to run agent decision %#v: %s", execution.ExecutionId, decision, err)
} else {
log.Printf("[ERROR][%s] AI Agent: Failed to run agent decision %#v: %s", execution.ExecutionId, decision.RunDetails.Id, err)
}
decision.RunDetails.Status = "FAILURE"
if len(decision.RunDetails.RawResponse) == 0 {
decision.RunDetails.RawResponse = fmt.Sprintf("Failed to start decision action. Raw Error: %s", err)
}
} else {
decision.RunDetails.Status = "FINISHED"
}
// Log individual tool execution result
duration := int64(0)
if decision.RunDetails.CompletedAt > 0 && decision.RunDetails.StartedAt > 0 {
duration = decision.RunDetails.CompletedAt - decision.RunDetails.StartedAt
}
log.Printf("[INFO][%s] AI_AGENT_TOOL: org=%s tool=%s action=%s status=%s duration=%ds", execution.ExecutionId, execution.Workflow.OrgId, decision.Tool, decision.Action, decision.RunDetails.Status, duration)
}
// 1. Send this back as a result for an action
// Then the action itself should decide if it's done or not.
// Would it work to send JUST this decision result?
// This could start the next step(s) automatically?
decision.RunDetails.CompletedAt = time.Now().UnixMilli()
marshalledDecision, err = json.Marshal(decision)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed marshalling completed decision %s", execution.ExecutionId, decision.RunDetails.Id)
}
go SetCache(ctx, decisionId, marshalledDecision, 300)
// 1. Send an /api/v1/streams request? Due to concurrency, I think this is the only way (?)
// 2. On the streams API, make sure to:
// 1. Check if the execution(s) are finished
// 2. Send the result through AI again to check if it changes (?). Should there be a verdict here?
// 3: Start the next steps of decisions after updates
baseUrl := "https://shuffler.io"
if os.Getenv("BASE_URL") != "" {
baseUrl = os.Getenv("BASE_URL")
}
if os.Getenv("SHUFFLE_CLOUDRUN_URL") != "" {
baseUrl = os.Getenv("SHUFFLE_CLOUDRUN_URL")
}
//url := fmt.Sprintf("%s/api/v1/apps/categories/run?authorization=%s&execution_id=%s", baseUrl, execution.Authorization, execution.ExecutionId)
url := fmt.Sprintf("%s/api/v1/streams", baseUrl)
log.Printf("[DEBUG][%s] Sending agent decision response %s with status %s. Node: %s. URL: %s", execution.ExecutionId, decision.RunDetails.Id, decision.RunDetails.Status, agentOutput.NodeId, url)
//?authorization=%s&execution_id=%s", baseUrl, execution.Authorization, execution.ExecutionId)
client := GetExternalClient(url)
// This is exactly how results for decisions as sent huh
// May need to do this for agentic question, answers as well
parsedAction := ActionResult{
ExecutionId: execution.ExecutionId,
Authorization: execution.Authorization,
// Map in the node ID (action ID) and decision ID to set/continue the right result
Action: Action{
AppName: "AI Agent",
Label: fmt.Sprintf("Agent Decision %s", decision.RunDetails.Id),
ID: agentOutput.NodeId,
},
Status: fmt.Sprintf("agent_%s", decision.RunDetails.Id),
Result: string(marshalledDecision),
}
for _, action := range execution.Workflow.Actions {
if action.ID == parsedAction.Action.ID {
parsedAction.Action = action
break
}
}
marshalledAction, err := json.Marshal(parsedAction)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed marshalling action in agent decision: %s", execution.ExecutionId, err)
return
}
req, err := http.NewRequest(
"POST",
url,
bytes.NewBuffer(marshalledAction),
)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed agent decision request creation: %s", execution.ExecutionId, err)
return
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed sending agent decision result: %s", execution.ExecutionId, err)
return
}
foundBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("[ERROR][%s] AI Agent: Failed reading body from agent decision: %s", execution.ExecutionId, err)
return
}
if resp.StatusCode != 200 {
log.Printf("[ERROR][%s] AI Agent: Status %d for decision %s. Body: %s", execution.ExecutionId, resp.StatusCode, decision.RunDetails.Id, string(foundBody))
}
}
func HandleCloudSyncAuthentication(resp http.ResponseWriter, request *http.Request) (SyncKey, error) {
apikey := request.Header.Get("Authorization")
if len(apikey) > 0 {
apikey = strings.Replace(apikey, " ", " ", -1)
if !strings.HasPrefix(apikey, "Bearer ") {
log.Printf("[WARNING] Apikey doesn't start with bearer: %s", apikey)
return SyncKey{}, errors.New("No bearer token for authorization header")
}
apikeyCheck := strings.Split(apikey, " ")
if len(apikeyCheck) != 2 {
log.Printf("[WARNING] Invalid format for apikey: %s", apikeyCheck)
return SyncKey{}, errors.New("Invalid format for apikey")
}
newApikey := apikeyCheck[1]
ctx := GetContext(request)
org, err := getSyncApikey(ctx, newApikey)
if err != nil {
log.Printf("[WARNING] Error in sync check: %s", err)
return SyncKey{}, errors.New(fmt.Sprintf("Error finding key: %s", err))
}
return SyncKey{Apikey: newApikey, OrgId: org}, nil
}
return SyncKey{}, errors.New("Missing authentication")
}
func HandleOrborusFailover(ctx context.Context, request *http.Request, resp http.ResponseWriter, env *Environment) error {
if len(env.Id) == 0 || len(env.Name) == 0 {
// Avoiding this onprem as it doesn't make sense
if project.Environment != "cloud" {
return nil
}
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Environment ID or Name is not set"}`)))
return errors.New("Environment ID or Name is not set")
}
orborusLabel := request.Header.Get("x-orborus-label")
var orboruserr error
var orborusData OrborusStats
body, bodyerr := ioutil.ReadAll(request.Body)
if bodyerr == nil {
orboruserr := json.Unmarshal(body, &orborusData)
if !env.SensorGroup && orboruserr == nil {
if time.Now().Unix() > env.Checkin+90 {
if debug {
log.Printf("[DEBUG] Failover orborus to '%s'. Checkin: %d. Edit: %d", orborusData.Uuid, env.Checkin, env.Edited)
}
env.OrborusUuid = orborusData.Uuid
}
if env.OrborusUuid != orborusData.Uuid && len(env.OrborusUuid) > 0 {
resp.WriteHeader(409)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Orborus UUID mismatch. This means another Orborus (Leader) is already handling this Runtime Location queue. This persists past a few minutes with only one Orborus running, please contact support@shuffler.io"}`)))
return errors.New("Orborus UUID mismatch")
} else {
//env.Checkin = time.Now().Unix()
}
}
}
// Handles a group of hosts running Orborus based on this page:
// https://security.shuffler.io/assets
if env.SensorGroup {
if len(orborusData.Uuid) == 0 || len(orborusData.SensorDetails.Hostname) == 0 {
if debug {
log.Printf("[DEBUG] Orborus data missing UUID or Hostname for sensor group environment '%s' (%s). Orborus Data: %#v", env.Name, env.Id, orborusData)
}
return nil
}
if strings.Contains(orborusData.SensorDetails.Hostname, ".") {
parsedHostnameSplit := strings.Split(orborusData.SensorDetails.Hostname, ".")
orborusData.SensorDetails.Hostname = parsedHostnameSplit[0]
}
// 1 month timeout before removed from the list. We only store
// minimal data anyway, so it really shouldn't matter
hostTimeout := int64(2592000)
hostRefresh := int64(90)
timeNow := int64(time.Now().Unix())
// Using cache to not have to constantly update the environment for every host
// This should fix itself over time (eventual completeness)
checkinKey := fmt.Sprintf("sensor_%s_%s_%s_checkin", env.Name, orborusData.SensorDetails.Hostname, orborusData.SensorDetails.Arch)
timeNowString := fmt.Sprintf("%d", timeNow)
SetCache(ctx, checkinKey, []byte(timeNowString), 120)
removeIndex := []int{}
found := false
updateMade := false
// Just some deduping in case
foundHosts := []string{}
for hostIndex, host := range env.SensorHosts {
parsedHost := fmt.Sprintf("%s-%s", host.Hostname, host.Arch)
if ArrayContains(foundHosts, parsedHost) {
removeIndex = append(removeIndex, hostIndex)
continue
}
foundHosts = append(foundHosts, parsedHost)
}
// Run removeIndex backwards
for i := len(removeIndex) - 1; i >= 0; i-- {
env.SensorHosts = append(env.SensorHosts[:removeIndex[i]], env.SensorHosts[removeIndex[i]+1:]...)
updateMade = true
}
removeIndex = []int{}
for hostIndex, host := range env.SensorHosts {
// Check if more than 90 seconds ago
if host.Hostname == orborusData.SensorDetails.Hostname && host.Arch == orborusData.SensorDetails.Arch {
found = true
if timeNow > host.Checkin+hostRefresh || env.SensorHosts[hostIndex].Uuid != orborusData.Uuid {
if debug {
//log.Printf("[DEBUG] Sensor '%s' in group environment '%s' (%s) is refreshing its checkin. Previous checkin: %d seconds ago", host.Hostname, env.Name, env.Id, timeNow-host.Checkin)
}
updateMade = true
env.SensorHosts[hostIndex].Checkin = timeNow
env.SensorHosts[hostIndex].Uuid = orborusData.Uuid
// FIXME: This needs to be a bit smarter
// For now we will just keep whatever we get first. Any restart
// of the agent will change it.
if host.Uuid != orborusData.Uuid {
DeleteCache(ctx, fmt.Sprintf("sensorupdate_%s_%s", orborusData.SensorDetails.Hostname, orborusData.SensorDetails.Arch))
env.SensorHosts[hostIndex].AutomaticScreenlockEnabled = orborusData.SensorDetails.AutomaticScreenlockEnabled
env.SensorHosts[hostIndex].HdEncrypted = orborusData.SensorDetails.HdEncrypted
env.SensorHosts[hostIndex].LogForwarding = orborusData.SensorDetails.LogForwarding
env.SensorHosts[hostIndex].ResponseActions = orborusData.SensorDetails.ResponseActions
if len(orborusData.SensorDetails.Serial) > 0 {
env.SensorHosts[hostIndex].Serial = orborusData.SensorDetails.Serial
}
if len(orborusData.SensorDetails.InstalledSoftware) > 0 {
env.SensorHosts[hostIndex].InstalledSoftware = orborusData.SensorDetails.InstalledSoftware
}
if len(orborusData.SensorDetails.InstalledSoftware) > 0 {
env.SensorHosts[hostIndex].CodeScanner = orborusData.SensorDetails.CodeScanner
}
}
}
break
}
}
// Appending a new one
if !found {
if debug {
log.Printf("\n\n[DEBUG] Adding new sensor host '%s' to group environment '%s' (%s). Total hosts: %d\n\n", orborusData.SensorDetails.Hostname, env.Name, env.Id, len(env.SensorHosts)+1)
}
updateMade = true
newHost := orborusData.SensorDetails
newHost.Uuid = orborusData.Uuid
newHost.Checkin = timeNow
env.SensorHosts = append(env.SensorHosts, newHost)
}
// Updates at that point (2 minutes~) as long as a single sensor is sending data.
if !updateMade && env.Checkin > 0 && timeNow > env.Checkin+120 {
updateMade = true
}
if updateMade && len(env.SensorHosts) >= 1 {
if debug {
//log.Printf("[DEBUG] Updating sensor host data for group environment '%s' (%s). Total hosts: %d. Checkin: %d seconds ago\n\n", env.Name, env.Id, len(env.SensorHosts), timeNow-env.Checkin)
}
// Sideloading from shuffle-security_sensors instead
removeIndex = []int{}
for sensorIndex, sensor := range env.SensorHosts {
if sensor.Hostname == orborusData.SensorDetails.Hostname && sensor.Arch == orborusData.SensorDetails.Arch {
sensor.Checkin = timeNow
orborusData.SensorDetails.Checkin = timeNow
} else {
checkinKeyCheck := fmt.Sprintf("sensor_%s_%s_%s_checkin", env.Name, sensor.Hostname, sensor.Arch)
foundCache, err := GetCache(ctx, checkinKeyCheck)
if err == nil {
cacheData := string(foundCache.([]uint8))
timestamp, err := strconv.Atoi(cacheData)
if err == nil && timestamp > 0 {
sensor.Checkin = int64(timestamp)
} else {
log.Printf("\n\n[ERROR] Failed ATOI for timestamp of %s: %s. Output: %s\n\n", sensor.Hostname, err, cacheData)
}
}
}
// Check if more than X minutes ago to time out hosts
if timeNow > sensor.Checkin+hostTimeout {
removeIndex = append(removeIndex, sensorIndex)
continue
}
// To reset the data that the env has
// as it doesn't need that much
env.SensorHosts[sensorIndex] = SensorDetails{
Hostname: sensor.Hostname,
Arch: sensor.Arch,
Uuid: sensor.Uuid,
Checkin: sensor.Checkin,
}
}
// Last cleanup
for i := len(removeIndex) - 1; i >= 0; i-- {
env.SensorHosts = append(env.SensorHosts[:removeIndex[i]], env.SensorHosts[removeIndex[i]+1:]...)
log.Printf("[INFO] Sensor '%s' removed from group environment '%s' (%s) due to inactivity. Checkin: %d seconds ago", env.SensorHosts[removeIndex[i]].Hostname, env.Name, env.Id, timeNow-env.SensorHosts[removeIndex[i]].Checkin)
}
go HandleSensorDatastoreUpdate(orborusData)
env.Checkin = timeNow
err := SetEnvironment(ctx, env)
if err != nil {
log.Printf("[ERROR] Sensor group environment '%s' (%s) FAILED to update with new sensor host data. Checkin: %d. Total hosts: %d. Error: %s", env.Name, env.Id, env.Checkin, len(env.SensorHosts), err)
}
}
return nil
}
timeNow := time.Now().Unix()
if request.Method == "POST" {
// Updates every 60 seconds~
if time.Now().Unix() > env.Checkin+60 {
// Print 1/10 times
if rand.Intn(10) == 0 {
log.Printf("[INFO] Updating environment '%s' (%s) from Orborus checkin (60 sec timeout). Previous checkin: %d seconds ago", env.Name, env.Id, timeNow-env.Checkin)
}
env.RunningIp = GetRequestIp(request)
// Orborus label = custom label for Orborus
if len(orborusLabel) > 0 {
env.RunningIp = orborusLabel
}
// Set the checkin cache
if bodyerr == nil && orboruserr == nil {
orborusData.RunningIp = env.RunningIp
env.OrborusUuid = orborusData.Uuid
marshalled, err := json.Marshal(orborusData)
if err == nil {
// Store for a full day. It's reset anyway in the UI at a certain point
cacheKey := fmt.Sprintf("queueconfig-%s-%s", env.Name, env.OrgId)
go SetCache(context.Background(), cacheKey, marshalled, 1440)
}
if orborusData.Swarm {
//env.Licensed = true
env.RunType = "docker"
}
if orborusData.Kubernetes {
env.RunType = "k8s"
}
orborusData.DataLake = env.DataLake
}
env.Checkin = timeNow
err := SetEnvironment(ctx, env)
if err != nil {
log.Printf("[ERROR] Failed updating environment: %s", err)
}
} else {
//if debug {
// log.Printf("[DEBUG] NOT updating env %s yet: %d seconds since checkin", env.Name, timeNow-env.Checkin)
//}
}
}
if env.Archived {
resp.WriteHeader(400)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't use archived environments. Make a new environment or restore the existing one."}`)))
return errors.New("Environment is archived")
}
return nil
}
// Sets sensor details in the org that they belong to
func HandleSensorDatastoreUpdate(orborusDetails OrborusStats) {
if len(orborusDetails.SensorDetails.Hostname) == 0 || orborusDetails.OrgId == "" {
if debug {
log.Printf("[DEBUG] Not updating datastore for sensor without hostname/orgId.")
}
return
}
sensorDetails := orborusDetails.SensorDetails
ctx := context.Background()
// MAX every 60 minutes, or if a sensor is restarted
cacheKey := fmt.Sprintf("sensorupdate_%s_%s", sensorDetails.Hostname, sensorDetails.Arch)
GotCache, err := GetCache(ctx, cacheKey)
if err == nil && GotCache != nil {
//if debug {
// log.Printf("[DEBUG] Skipping datastore update for sensor '%s' as it was updated recently (cache hit)", sensorDetails.Hostname)
//}
return
}
// 24 hour updates. Don't want to overload it.
SetCache(ctx, cacheKey, []byte("1"), 1440)
datastoreSensorIndex := "shuffle-security_sensors"
datastorePackageIndex := "shuffle-security_packages"
// Sets the current sensor details raw
sensorDetails.Checkin = time.Now().Unix()
parsedHostname := strings.TrimSpace(strings.ReplaceAll(strings.ToUpper(sensorDetails.Hostname), " ", "_"))
skippedAmount := 0
maxSoftwareAmount := 1000
handledKeys := []string{}
softwareWg := sync.WaitGroup{}
datastoreSoftwareIndex := "shuffle-security_software"
softwareAmount := len(sensorDetails.InstalledSoftware)
if softwareAmount > maxSoftwareAmount {
softwareAmount = maxSoftwareAmount
}
softwareKeys := make(chan CacheKeyData, softwareAmount)
for softwareCnt, software := range sensorDetails.InstalledSoftware {
if softwareCnt+skippedAmount > maxSoftwareAmount {
break
}
// linux/macos/windows handler
if softwareCnt == 0 && strings.Contains(software.Name, " ") {
software.Name = strings.ReplaceAll(software.Name, software.Version, "")
nameSplit := strings.Split(software.Name, " ")
software.Name = nameSplit[0]
for _, part := range nameSplit[1:] {
if len(part) <= 1 {
continue
}
software.Version = fmt.Sprintf("%s-%s", software.Version, part)
}
}
parsedKeyname := fmt.Sprintf("%s_%s", strings.TrimSpace(strings.ReplaceAll(strings.ToLower(software.Name), " ", "_")), sensorDetails.OS)
if ArrayContains(handledKeys, software.Name) {
skippedAmount += 1
continue
}
handledKeys = append(handledKeys, software.Name)
softwareWg.Add(1)
go func(parsedKeyname string, software Software) {
defer softwareWg.Done()
// 1. Get existing key
// 2. Update Versions & Hostnames
// 3. If it existed already, don't update the "Last Seen" field (or set it to the oldest of the two)
software.OS = sensorDetails.OS
software.Hostnames = []HostDetails{
HostDetails{
Hostname: sensorDetails.Hostname,
Version: software.Version,
UpdatedAt: time.Now().Unix(),
},
}
if len(software.Version) > 0 {
software.Versions = []string{software.Version}
}
datastoreId := fmt.Sprintf("%s_%s_%s", orborusDetails.OrgId, parsedKeyname, datastoreSoftwareIndex)
config, getCacheError := GetDatastoreKey(ctx, datastoreId, datastoreSoftwareIndex)
if getCacheError != nil {
//log.Printf("[ERROR] Failed to get existing datastore key for software '%s': %s", parsedKeyname, getCacheError)
} else if len(config.Value) > 0 {
unmarshalledSoftware := Software{}
err := json.Unmarshal([]byte(config.Value), &unmarshalledSoftware)
if err == nil {
hostExists := false
versionExists := false
for _, foundHost := range unmarshalledSoftware.Hostnames {
if foundHost.Hostname == sensorDetails.Hostname && foundHost.Version == software.Version {
hostExists = true
break
}
}
if !hostExists {
unmarshalledSoftware.Hostnames = append(unmarshalledSoftware.Hostnames, HostDetails{
Hostname: sensorDetails.Hostname,
Version: software.Version,
UpdatedAt: time.Now().Unix(),
})
}
if ArrayContains(unmarshalledSoftware.Versions, software.Version) {
versionExists = true
} else {
unmarshalledSoftware.Versions = append(unmarshalledSoftware.Versions, software.Version)
}
if hostExists && versionExists {
if debug {
//log.Printf("[DEBUG] Software '%s' on host '%s' with version '%s' already exists in datastore. Skipping update.", software.Name, sensorDetails.Hostname, software.Version)
}
softwareKeys <- CacheKeyData{
Key: "",
}
return
}
software = unmarshalledSoftware
}
}
software.Version = ""
parsedValue, err := json.Marshal(software)
if err != nil {
log.Printf("[ERROR] Failed to marshal software for datastore update: %s. Software: %#v", err, software)
softwareKeys <- CacheKeyData{
Key: "",
}
return
}
newKey := CacheKeyData{
Key: parsedKeyname,
Category: datastoreSoftwareIndex,
Value: string(parsedValue),
OrgId: orborusDetails.OrgId,
}
softwareKeys <- newKey
}(parsedKeyname, software)
}
packageAmount := 0
handledKeys = []string{}
for _, curPackage := range sensorDetails.CodeScanner {
if packageAmount > maxSoftwareAmount {
break
}
// Dedups
for _, software := range curPackage.Packages {
if packageAmount > maxSoftwareAmount {
break
}
parsedKeyname := strings.TrimSpace(strings.ReplaceAll(strings.ToLower(software.Name), " ", "_"))
if ArrayContains(handledKeys, parsedKeyname) {
skippedAmount += 1
continue
}
handledKeys = append(handledKeys, parsedKeyname)
packageAmount += 1
}
}
skippedAmount = 0
handledKeys = []string{}
packageWg := sync.WaitGroup{}
packageKeys := make(chan CacheKeyData, packageAmount)
totalCount := 0
for _, curPackage := range sensorDetails.CodeScanner {
if totalCount >= maxSoftwareAmount {
log.Printf("[WARNING] Reached max amount of software+packages to update for sensor '%s'. Total count: %d. Skipped amount: %d", sensorDetails.Hostname, totalCount, skippedAmount)
break
}
// Loop the inner part
for _, software := range curPackage.Packages {
if totalCount >= maxSoftwareAmount {
break
}
//parsedKeyname := fmt.Sprintf("%s_%s", strings.TrimSpace(strings.ReplaceAll(strings.ToLower(software.Name), " ", "_")), sensorDetails.OS)
parsedKeyname := strings.TrimSpace(strings.ReplaceAll(strings.ToLower(software.Name), " ", "_"))
if ArrayContains(handledKeys, parsedKeyname) {
skippedAmount += 1
continue
}
handledKeys = append(handledKeys, parsedKeyname)
packageWg.Add(1)
go func(parsedKeyname string, software Software) {
defer packageWg.Done()
// 1. Get existing key
// 2. Update Versions & Hostnames
// 3. If it existed already, don't update the "Last Seen" field (or set it to the oldest of the two)
software.OS = curPackage.Type
software.Hostnames = []HostDetails{
HostDetails{
Hostname: sensorDetails.Hostname,
Version: software.Version,
UpdatedAt: time.Now().Unix(),
Paths: []string{curPackage.Path},
},
}
if len(software.Version) > 0 {
software.Versions = []string{software.Version}
}
datastoreId := fmt.Sprintf("%s_%s_%s", orborusDetails.OrgId, parsedKeyname, datastorePackageIndex)
config, getCacheError := GetDatastoreKey(ctx, datastoreId, datastorePackageIndex)
if getCacheError != nil {
//log.Printf("[ERROR] Failed to get existing datastore key for software '%s': %s", parsedKeyname, getCacheError)
} else if len(config.Value) > 0 {
unmarshalledSoftware := Software{}
err := json.Unmarshal([]byte(config.Value), &unmarshalledSoftware)
if err == nil {
hostPathExists := false
versionExists := false
for foundHostIndex, foundHost := range unmarshalledSoftware.Hostnames {
if foundHost.Hostname == sensorDetails.Hostname && foundHost.Version == software.Version {
unmarshalledSoftware.Hostnames[foundHostIndex].UpdatedAt = time.Now().Unix()
found := false
for _, path := range unmarshalledSoftware.Hostnames[foundHostIndex].Paths {
if path == curPackage.Path {
unmarshalledSoftware.Hostnames[foundHostIndex].Paths = append(unmarshalledSoftware.Hostnames[foundHostIndex].Paths, path)
found = true
break
}
}
if !found {
hostPathExists = true
}
break
}
}
if !hostPathExists {
unmarshalledSoftware.Hostnames = append(unmarshalledSoftware.Hostnames, HostDetails{
Hostname: sensorDetails.Hostname,
Version: software.Version,
UpdatedAt: time.Now().Unix(),
Paths: []string{curPackage.Path},
})
}
if ArrayContains(unmarshalledSoftware.Versions, software.Version) {
versionExists = true
} else {
unmarshalledSoftware.Versions = append(unmarshalledSoftware.Versions, software.Version)
}
if hostPathExists && versionExists {
if debug {
//log.Printf("[DEBUG] Package '%s' on host '%s' with version '%s' already exists in datastore. Skipping update.", software.Name, sensorDetails.Hostname, software.Version)
}
packageKeys <- CacheKeyData{
Key: "",
}
return
}
software = unmarshalledSoftware
}
}
software.Version = ""
parsedValue, err := json.Marshal(software)
if err != nil {
log.Printf("[ERROR] Failed to marshal Package for datastore update: %s. Software: %#v", err, curPackage)
packageKeys <- CacheKeyData{
Key: "",
}
return
}
packageKeys <- CacheKeyData{
Key: parsedKeyname,
Category: datastorePackageIndex,
Value: string(parsedValue),
OrgId: orborusDetails.OrgId,
}
}(parsedKeyname, software)
totalCount += 1
}
}
softwareWg.Wait()
close(softwareKeys)
packageWg.Wait()
close(packageKeys)
// Doing another dedup here as well
newPackageArray := []CacheKeyData{}
for key := range packageKeys {
if key.Key == "" {
continue
}
found := false
for packageIndex, newPackage := range newPackageArray {
if newPackage.Key != key.Key {
continue
}
log.Printf("[DEBUG] FOUND DUPE: %s", key.Key)
found = true
unmarshalledSoftwareNew := Software{}
err := json.Unmarshal([]byte(key.Value), &unmarshalledSoftwareNew)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal software for package deduplication: %s. Software: %#v", err, key.Value)
continue
}
unmarshalledSoftwareExisting := Software{}
err = json.Unmarshal([]byte(newPackage.Value), &unmarshalledSoftwareExisting)
if err != nil {
log.Printf("[ERROR] Failed to unmarshal software for package deduplication: %s. Software: %#v", err, newPackage.Value)
continue
}
// Make sure the path and version exists
updated := false
for _, newHost := range unmarshalledSoftwareNew.Hostnames {
if !ArrayContains(unmarshalledSoftwareExisting.Versions, newHost.Version) {
continue
}
existingHostIndex := -1
for i, existingHost := range unmarshalledSoftwareExisting.Hostnames {
if existingHost.Hostname == newHost.Hostname {
existingHostIndex = i
break
}
}
if existingHostIndex == -1 {
unmarshalledSoftwareExisting.Hostnames = append(unmarshalledSoftwareExisting.Hostnames, newHost)
} else {
for _, newPath := range newHost.Paths {
if !ArrayContains(unmarshalledSoftwareExisting.Hostnames[existingHostIndex].Paths, newPath) {
unmarshalledSoftwareExisting.Hostnames[existingHostIndex].Paths = append(unmarshalledSoftwareExisting.Hostnames[existingHostIndex].Paths, newPath)
updated = true
}
}
}
// Update the "Last Seen" field to be the oldest of the two
if unmarshalledSoftwareExisting.Hostnames[existingHostIndex].UpdatedAt < newHost.UpdatedAt {
unmarshalledSoftwareExisting.Hostnames[existingHostIndex].UpdatedAt = newHost.UpdatedAt
updated = true
}
}
// FIXME: SOMETHING is wrong here.
if updated {
if debug {
log.Printf("FOUND DUPE: %s. Updated existing package key with new host and paths. %#v", key.Key, unmarshalledSoftwareExisting)
log.Printf("Old value: %#v", newPackage.Value)
log.Printf("New value: %#v", key.Value)
}
parsedValue, err := json.Marshal(unmarshalledSoftwareExisting)
if err != nil {
log.Printf("[ERROR] Failed to marshal software for package deduplication update: %s. Software: %#v", err, unmarshalledSoftwareExisting)
continue
}
newPackageArray[packageIndex].Value = string(parsedValue)
}
}
// We need to deduplicate here
if !found {
newPackageArray = append(newPackageArray, key)
}
}
newSoftwareArray := []CacheKeyData{}
for key := range softwareKeys {
if key.Key == "" {
continue
}
newSoftwareArray = append(newSoftwareArray, key)
}
if debug {
log.Printf("[DEBUG] %s - Packages: %d. Software: %d. Skipped amount: %d", sensorDetails.Hostname, len(newPackageArray), len(newSoftwareArray), skippedAmount)
}
if len(newSoftwareArray) > 0 {
if debug {
log.Printf("[DEBUG] Updating datastore with %d software keys for sensor '%s'", len(newSoftwareArray), sensorDetails.Hostname)
}
// Set them in the datastore (with some delay to avoid spikes)
_, err = SetDatastoreKeyBulk(ctx, newSoftwareArray)
if err != nil {
log.Printf("[ERROR] Failed to update datastore with %d software keys for sensor '%s': %s", len(newSoftwareArray), sensorDetails.Hostname, err)
}
}
if len(newPackageArray) > 0 {
if debug {
log.Printf("[DEBUG] Updating datastore with %d package keys for sensor '%s'", len(newPackageArray), sensorDetails.Hostname)
}
// Set them in the datastore (with some delay to avoid spikes)
_, err = SetDatastoreKeyBulk(ctx, newPackageArray)
if err != nil {
log.Printf("[ERROR] Failed to update datastore with %d package keys for sensor '%s': %s", len(newPackageArray), sensorDetails.Hostname, err)
}
}
// Loading in historical info
// Putting it here so we don't re-upload without a reason
if len(sensorDetails.CodeScanner) == 0 || len(sensorDetails.CodeScanner) == 0 {
datastoreId := fmt.Sprintf("%s_%s_%s", orborusDetails.OrgId, parsedHostname, datastoreSensorIndex)
cachedHost, err := GetDatastoreKey(ctx, datastoreId, datastoreSensorIndex)
if err == nil && len(cachedHost.Value) > 0 {
// unmarshal value to check what exists
oldHost := SensorDetails{}
err := json.Unmarshal([]byte(cachedHost.Value), &oldHost)
if err == nil {
if len(oldHost.User) > 0 && len(sensorDetails.User) == 0 {
sensorDetails.User = oldHost.User
}
if len(oldHost.CodeScanner) > 0 {
sensorDetails.CodeScanner = oldHost.CodeScanner
}
if len(oldHost.InstalledSoftware) > 0 {
sensorDetails.InstalledSoftware = oldHost.InstalledSoftware
}
}
} else {
log.Printf("[ERROR] Failed to load existing sensor details for sensor '%s' from datastore: %s", sensorDetails.Hostname, err)
}
}
sensorDetails.Checkin = time.Now().Unix()
hostData, err := json.Marshal(sensorDetails)
if err != nil {
log.Printf("[ERROR] Failed to marshal sensor details for datastore update for sensor '%s': %s", sensorDetails.Hostname, err)
} else {
hostKey := CacheKeyData{
Key: parsedHostname,
Category: datastoreSensorIndex,
Value: string(hostData),
OrgId: orborusDetails.OrgId,
}
// Set them in the datastore (with some delay to avoid spikes)
_, err = SetDatastoreKeyBulk(ctx, []CacheKeyData{hostKey})
if err != nil {
log.Printf("[ERROR] Failed to update datastore with software keys for sensor '%s': %s", sensorDetails.Hostname, err)
}
}
}
// Download handler for Orborus agent installation script. This is used in the "Assets" page for Orborus, and can be used by customers to easily install Orborus on their hosts. It returns a bash script that can be run on the target host to install Orborus with the correct configuration.
func GetOrborusDownloadCommand(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
// 1. fetch config in Go (no jq dependency)
c := OrborusDownloadConfig{
BaseURL: "https://shuffler.io",
Queue: "default",
Auth: "cb5st3d3Z!3X3zaJ*Pc",
OrgID: "",
SoftwareListEnabled: true,
HDEncryptedCheck: true,
ScreenlockCheck: true,
ResponseActions: "full",
AsRoot: true,
// Used for builder in dynamic scripts
BinaryBaseURL: "https://github.com/Shuffle/orborus/releases/latest/download",
Binaries: map[string]string{
"linux_amd64": "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-linux-amd64",
"linux_arm64": "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-linux-arm64",
"darwin_amd64": "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-darwin-amd64",
"darwin_arm64": "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-darwin-arm64",
"windows_amd64": "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-windows-amd64.exe",
"windows_arm64": "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-windows-arm64.exe",
},
}
// 2. URL overrides (optional)
q := r.URL.Query()
isWindows := false
if v := q.Get("os"); v != "" {
if v == "windows" {
isWindows = true
}
}
if v := q.Get("base_url"); v != "" {
c.BaseURL = v
}
if v := q.Get("queue"); v != "" {
c.Queue = v
}
if v := q.Get("auth"); v != "" {
c.Auth = v
}
if v := q.Get("org_id"); v != "" {
c.OrgID = v
}
if v := q.Get("response_actions"); v != "" {
c.ResponseActions = v
}
if v := q.Get("log_forwarding"); v != "" {
c.LogForwarding = v
}
if v := q.Get("software_list_enabled"); v != "" {
c.SoftwareListEnabled = v == "true"
}
if v := q.Get("hd_encrypted_check"); v != "" {
c.HDEncryptedCheck = v == "true"
}
if v := q.Get("screenlock_check"); v != "" {
c.ScreenlockCheck = v == "true"
}
if v := q.Get("admin"); v != "" {
c.AsRoot = v != "false"
}
// Check the "AUTH" header for a secret value to allow overriding the config (for security)
if authHeader := r.Header.Get("AUTH"); authHeader != "" {
c.Auth = authHeader
}
// Quite untested.
script := ""
if isWindows {
script = fmt.Sprintf(`[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$principal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())
$isAdmin = $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
if (-not $isAdmin) {
Start-Process powershell -Verb RunAs -ArgumentList @(
"-NoProfile",
"-ExecutionPolicy",
"Bypass",
"-Command",
"iwr <your-url> | iex"
)
Write-Host "started sensor installation in a new elevated PowerShell window. Please follow the prompts there to complete installation.",
exit 1
}
$ErrorActionPreference = "Stop"
# ===== injected config (from cfg) =====
$BASE_URL = "%s"
$QUEUE = "%s"
$AUTH = "%s"
$ORG_ID = "%s"
$SOFTWARE_LIST_ENABLED = "%t"
$CODE_SCANNER_ENABLED = "%t"
$HD_ENCRYPTED_CHECK = "%t"
$SCREENLOCK_CHECK = "%t"
$RESPONSE_ACTIONS = "%s"
$LOG_FORWARDING = "%s"
# ===== install paths =====
$INSTALL_DIR = "$env:ProgramData\orborus"
New-Item -ItemType Directory -Force -Path $INSTALL_DIR | Out-Null
# ===== arch detection =====
if ($env:PROCESSOR_ARCHITECTURE -eq "AMD64") {
$ARCH = "amd64"
} elseif ($env:PROCESSOR_ARCHITECTURE -eq "ARM64") {
$ARCH = "arm64"
} else {
Write-Error "Unsupported architecture: $env:PROCESSOR_ARCHITECTURE"
exit 1
}
$BIN_URL = "https://github.com/Shuffle/orborus/releases/latest/download/orborus-agent-windows-$ARCH.exe"
$BIN_PATH = Join-Path $INSTALL_DIR "orborus-agent.exe"
icacls $INSTALL_DIR /grant Users:R
icacls $BIN_PATH /grant Users:RX
# Remove if exists early as the process may be running and we need to change the file
$SERVICE_NAME = "orborus-agent"
echo "Deleting old sensor"
schtasks /Delete /TN $SERVICE_NAME /F
$p = Get-Process -Name "orborus-agent" -ErrorAction SilentlyContinue
if ($p) {
$p | Stop-Process -Force
}
Start-Sleep -Seconds 2
Write-Host "Downloading binary from $BIN_URL to $BIN_PATH..."
try {
Write-Host "Downloading via BITS..."
Start-BitsTransfer -Source $BIN_URL -Destination $BIN_PATH -ErrorAction Stop
}
catch {
Write-Host "BITS failed, falling back to Invoke-WebRequest..."
Invoke-WebRequest -Uri $BIN_URL -OutFile $BIN_PATH -UseBasicParsing -MaximumRedirection 10
}
icacls $INSTALL_DIR /grant Users:R
icacls $BIN_PATH /grant Users:RX
# ===== service =====
function Escape-ArgValue($v) {
if ($v -match "\s") {
return '"' + $v + '"'
}
return $v
}
$ARGS = @()
$ARGS += "--sensor_mode=true"
if ($BASE_URL) { $ARGS += "--base_url=$BASE_URL" }
if ($QUEUE) { $ARGS += "--queue=$QUEUE" }
if ($AUTH) { $ARGS += "--auth=$AUTH" }
if ($ORG_ID) { $ARGS += "--org_id=$ORG_ID" }
# Removed as they made the command more than 260 characters (hard limit)
# These are now being enabled by default.
#if ($SOFTWARE_LIST_ENABLED -eq "true") { $ARGS += "--software_list_enabled=true" }
#if ($SOFTWARE_LIST_ENABLED -eq "false") { $ARGS += "--software_list_enabled=false" }
#if ($HD_ENCRYPTED_CHECK -eq "true") { $ARGS += "--hd_encrypted_check=true" }
#if ($SCREENLOCK_CHECK -eq "true") { $ARGS += "--screenlock_check=true" }
if ($RESPONSE_ACTIONS) { $ARGS += "--response_actions=$RESPONSE_ACTIONS" }
if ($LOG_FORWARDING) { $ARGS += "--log_forwarding=$LOG_FORWARDING" }
for ($i = 0; $i -lt $ARGS.Count; $i++) {
if ($ARGS[$i] -match '=') {
$parts = $ARGS[$i] -split '=', 2
$key = $parts[0]
$val = $parts[1]
if ($val -match '\s' -and $val -notmatch '^".*"$') {
$val = '"' + $val + '"'
}
$ARGS[$i] = "$key=$val"
}
}
$ARGS = $ARGS -join " "
# ===== create service =====
$WRAPPER = Join-Path $INSTALL_DIR "run-orborus.bat"
## Give exec permissions as user
icacls $WRAPPER /grant Users:RX
echo "Writing bat file to $WRAPPER"
$writer = New-Item -ItemType File -Path $Wrapper -Force
# Pre-prep
$line2 = "cd /d " + '"' + $INSTALL_DIR + '"'
$line4 = 'start "" ' + '"' + $BIN_PATH + '"' + " " + $ARGS + " >> orborus.log 2>&1"
Add-Content $WRAPPER "@echo off"
Add-Content $WRAPPER $line2
Add-Content $WRAPPER "echo STARTED >> debug.log"
Add-Content $WRAPPER $line4
Add-Content $WRAPPER "echo EXIT CODE %sRRORLEVEL%s >> debug.log"
echo "Starting scheduled task"
$PARSED_WRAPPER = '"' + $WRAPPER + '"'
# IF you want to run it without admin permissions, don't set /RU SYSTEM here
# Problem is then it's controllable by users too. That's fine for now.
$RUN_AS_ROOT = "%t"
if ($RUN_AS_ROOT -eq "true") {
echo "Running as root (default) - admin=false to disable"
schtasks /Create /TN $SERVICE_NAME /TR "$PARSED_WRAPPER" /SC ONSTART /RU "SYSTEM" /RL HIGHEST /F
} else {
echo "Running as normal $env:USERNAME"
schtasks /Create /TN $SERVICE_NAME /TR "$PARSED_WRAPPER" /SC ONSTART /RL HIGHEST /F
}
echo "Running service"
schtasks /Run /TN $SERVICE_NAME
Write-Host "orborus-agent installed"`,
c.BaseURL,
c.Queue,
c.Auth,
c.OrgID,
c.SoftwareListEnabled,
c.CodeScannerEnabled,
c.HDEncryptedCheck,
c.ScreenlockCheck,
c.ResponseActions,
c.LogForwarding,
"%E",
"%",
c.AsRoot,
)
} else {
script = fmt.Sprintf(`#!/usr/bin/env bash
set -e
#if [ "$(id -u)" -ne 0 ]; then
# echo "Run installer as root."
# exit 1
#fi
# =========================
# Detect OS + ARCH
# =========================
OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
ARCH="$(uname -m)"
case "$ARCH" in
x86_64|amd64) ARCH="amd64" ;;
arm64|aarch64) ARCH="arm64" ;;
*)
echo "unsupported architecture: $ARCH"
exit 1
;;
esac
if [[ "$OS" != "linux" && "$OS" != "darwin" ]]; then
echo "unsupported OS: $OS"
exit 1
fi
echo ""
echo "Download started. Please be patient while we install the sensor. Detected OS: $OS, ARCH: $ARCH"
echo ""
# =========================
# Config (from Go injection)
# =========================
BASE_URL="%s"
QUEUE="%s"
AUTH="%s"
ORG_ID="%s"
SOFTWARE_LIST_ENABLED="%t"
CODE_SCANNER_ENABLED="%t"
HD_ENCRYPTED_CHECK="%t"
SCREENLOCK_CHECK="%t"
RESPONSE_ACTIONS="%s"
LOG_FORWARDING="%s"
# =========================
# Binary selection
# =========================
BIN_BASE="%s"
BIN_URL="${BIN_BASE}/orborus-agent-${OS}-${ARCH}"
# =========================
# Install binary
# =========================
INSTALL_PATH="/usr/local/bin/orborus"
echo "Download starting from $BIN_URL... This may take a minute."
curl -fsSL "$BIN_URL" -o /tmp/orborus
chmod +x /tmp/orborus
echo "If prompted, please input your sudo password to allow installation (required for service setup and sensor capabilities). Contact support@shuffler.io if you need help."
sudo mv /tmp/orborus "$INSTALL_PATH"
echo "Installed binary to $INSTALL_PATH"
# =========================
# Linux service (systemd)
# =========================
install_linux() {
sudo tee /etc/systemd/system/orborus.service > /dev/null <<EOF
[Unit]
Description=Orborus Agent
After=network.target
[Service]
Type=simple
ExecStart=$INSTALL_PATH \
--sensor_mode=true \
--base_url=$BASE_URL \
--queue=$QUEUE \
--auth=$AUTH \
--org_id=$ORG_ID \
--software_list_enabled=$SOFTWARE_LIST_ENABLED \
--code_scanner_enabled=$CODE_SCANNER_ENABLED \
--hd_encrypted_check=$HD_ENCRYPTED_CHECK \
--screenlock_check=$SCREENLOCK_CHECK \
--log_forwarding=$LOG_FORWARDING \
--response_actions=$RESPONSE_ACTIONS
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
sudo systemctl daemon-reload
sudo systemctl enable orborus
sudo systemctl restart orborus
}
# =========================
# macOS service (launchd)
# =========================
install_macos() {
PLIST=~/Library/LaunchAgents/com.orborus.agent.plist
mkdir -p ~/Library/LaunchAgents
cat > "$PLIST" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.orborus.agent</string>
<key>ProgramArguments</key>
<array>
<string>$INSTALL_PATH</string>
<string>--sensor_mode=true</string>
<string>--base_url=$BASE_URL</string>
<string>--queue=$QUEUE</string>
<string>--auth=$AUTH</string>
<string>--org_id=$ORG_ID</string>
<string>--software_list_enabled=$SOFTWARE_LIST_ENABLED</string>
<string>--code_scanner_enabled=$CODE_SCANNER_ENABLED</string>
<string>--hd_encrypted_check=$HD_ENCRYPTED_CHECK</string>
<string>--screenlock_check=$SCREENLOCK_CHECK</string>
<string>--log_forwarding=$LOG_FORWARDING</string>
<string>--response_actions=$RESPONSE_ACTIONS</string>
</array>
<key>RunAtLoad</key>
<true/>
</dict>
</plist>
EOF
launchctl unload "$PLIST" 2>/dev/null || true
launchctl load "$PLIST"
}
# =========================
# Execute
# =========================
if [ "$OS" = "linux" ]; then
install_linux
echo "orborus installed successfully"
elif [ "$OS" = "darwin" ]; then
install_macos
echo "orborus installed successfully"
fi
`,
c.BaseURL,
c.Queue,
c.Auth,
c.OrgID,
c.SoftwareListEnabled,
c.CodeScannerEnabled,
c.HDEncryptedCheck,
c.ScreenlockCheck,
c.ResponseActions,
c.LogForwarding,
c.BinaryBaseURL,
)
}
w.Write([]byte(script))
}
Reference in New Issue View Git Blame Copy Permalink