Merge branch 'nightly' of https://github.com/Shuffle/Shuffle into release-changes
This commit is contained in:
@@ -1,3 +1,11 @@
|
||||
# No extra requirements needed
|
||||
requests
|
||||
urllib3
|
||||
requests==2.32.3
|
||||
urllib3==2.3.0
|
||||
liquidpy==0.8.2
|
||||
MarkupSafe==3.0.2
|
||||
flask[async]==3.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
PyJWT==2.10.1
|
||||
cryptography==44.0.2
|
||||
shufflepy==0.1.0
|
||||
shuffle-sdk==0.0.25
|
||||
|
||||
+79
-68
@@ -211,7 +211,7 @@ func fixTags(tags []string) []string {
|
||||
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
|
||||
ctx := context.Background()
|
||||
client, err := client.NewEnvClient()
|
||||
defer client.Close()
|
||||
defer client.Close()
|
||||
if err != nil {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
return err
|
||||
@@ -473,73 +473,84 @@ func buildImage(tags []string, dockerfileLocation string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := client.NewEnvClient()
|
||||
defer client.Close()
|
||||
if err != nil {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("[INFO] Docker Tags: %s", tags)
|
||||
dockerfileSplit := strings.Split(dockerfileLocation, "/")
|
||||
|
||||
// Create a buffer
|
||||
buf := new(bytes.Buffer)
|
||||
tw := tar.NewWriter(buf)
|
||||
defer tw.Close()
|
||||
baseDir := strings.Join(dockerfileSplit[0:len(dockerfileSplit)-1], "/")
|
||||
|
||||
// Builds the entire folder into buf
|
||||
err = getParsedTar(tw, baseDir, "")
|
||||
if err != nil {
|
||||
log.Printf("Tar issue: %s", err)
|
||||
}
|
||||
|
||||
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
||||
buildOptions := types.ImageBuildOptions{
|
||||
Remove: true,
|
||||
Tags: tags,
|
||||
BuildArgs: map[string]*string{},
|
||||
}
|
||||
//NetworkMode: "host",
|
||||
|
||||
httpProxy := os.Getenv("HTTP_PROXY")
|
||||
if len(httpProxy) > 0 {
|
||||
buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy
|
||||
}
|
||||
httpsProxy := os.Getenv("HTTPS_PROXY")
|
||||
if len(httpProxy) > 0 {
|
||||
buildOptions.BuildArgs["https_proxy"] = &httpsProxy
|
||||
}
|
||||
|
||||
// Build the actual image
|
||||
imageBuildResponse, err := client.ImageBuild(
|
||||
ctx,
|
||||
dockerFileTarReader,
|
||||
buildOptions,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read the STDOUT from the build process
|
||||
defer imageBuildResponse.Body.Close()
|
||||
buildBuf := new(strings.Builder)
|
||||
_, err = io.Copy(buildBuf, imageBuildResponse.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
} else {
|
||||
if strings.Contains(buildBuf.String(), "errorDetail") {
|
||||
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
|
||||
return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ",")))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := client.NewEnvClient()
|
||||
defer client.Close()
|
||||
if err != nil {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("[INFO] Docker Tags: %s", tags)
|
||||
dockerfileSplit := strings.Split(dockerfileLocation, "/")
|
||||
|
||||
// Create a buffer
|
||||
buf := new(bytes.Buffer)
|
||||
tw := tar.NewWriter(buf)
|
||||
defer tw.Close()
|
||||
baseDir := strings.Join(dockerfileSplit[0:len(dockerfileSplit)-1], "/")
|
||||
|
||||
// Builds the entire folder into buf
|
||||
err = getParsedTar(tw, baseDir, "")
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Tar issue during app build: %s", err)
|
||||
}
|
||||
|
||||
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
||||
buildOptions := types.ImageBuildOptions{
|
||||
Remove: true,
|
||||
Tags: tags,
|
||||
BuildArgs: map[string]*string{},
|
||||
}
|
||||
//NetworkMode: "host",
|
||||
|
||||
httpProxy := os.Getenv("HTTP_PROXY")
|
||||
if len(httpProxy) > 0 {
|
||||
buildOptions.BuildArgs["HTTP_PROXY"] = &httpProxy
|
||||
}
|
||||
httpsProxy := os.Getenv("HTTPS_PROXY")
|
||||
if len(httpProxy) > 0 {
|
||||
buildOptions.BuildArgs["https_proxy"] = &httpsProxy
|
||||
}
|
||||
|
||||
// Print the actual file content from dockerFileTarReader
|
||||
/*
|
||||
data, err := ioutil.ReadAll(dockerFileTarReader)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed reading Dockerfile TAR reader: %s", err)
|
||||
} else {
|
||||
log.Printf("[DEBUG] Dockerfile TAR reader content: %s", string(data))
|
||||
}
|
||||
*/
|
||||
|
||||
// Build the actual image
|
||||
imageBuildResponse, err := client.ImageBuild(
|
||||
ctx,
|
||||
dockerFileTarReader,
|
||||
buildOptions,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read the STDOUT from the build process
|
||||
defer imageBuildResponse.Body.Close()
|
||||
buildBuf := new(strings.Builder)
|
||||
_, err = io.Copy(buildBuf, imageBuildResponse.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
} else {
|
||||
if strings.Contains(buildBuf.String(), "errorDetail") {
|
||||
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
|
||||
return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ",")))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -671,7 +682,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "No image name"}`)))
|
||||
return
|
||||
|
||||
|
||||
}
|
||||
|
||||
log.Printf("[INFO] Trying to download image: '%s'. Appname: '%s'. BaseAppname: '%s', Split2: %s", version.Name, appname, baseAppname, appnameSplit2)
|
||||
@@ -870,7 +881,7 @@ func handleRemoteDownloadApp(resp http.ResponseWriter, ctx context.Context, user
|
||||
type tmpapp struct {
|
||||
Success bool `json:"success"`
|
||||
OpenAPI string `json:"openapi"`
|
||||
App string `json:"app"`
|
||||
App string `json:"app"`
|
||||
}
|
||||
|
||||
app := tmpapp{}
|
||||
|
||||
@@ -22,7 +22,7 @@ require (
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shuffle/shuffle-shared v0.8.58
|
||||
github.com/shuffle/shuffle-shared v0.8.72
|
||||
golang.org/x/crypto v0.37.0
|
||||
google.golang.org/api v0.228.0
|
||||
google.golang.org/grpc v1.71.1
|
||||
|
||||
@@ -341,8 +341,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible h1:KDSasSTktAqMJCYClHVE94Fc
|
||||
github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8=
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/shuffle/shuffle-shared v0.8.58 h1:QzCKtQjuaozb+xyLkw6IjwqCNVWzpuJJnxkpXkLCP9M=
|
||||
github.com/shuffle/shuffle-shared v0.8.58/go.mod h1:OLAwH/Ym4941Jn5DF1oZaq6iBpmjG2SNrTZ9Xqck5So=
|
||||
github.com/shuffle/shuffle-shared v0.8.72 h1:HVOsRt83/1k9P+8q1FAxXnDKyROoDAFa1A3MnoRJYb0=
|
||||
github.com/shuffle/shuffle-shared v0.8.72/go.mod h1:OLAwH/Ym4941Jn5DF1oZaq6iBpmjG2SNrTZ9Xqck5So=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
|
||||
+101
-59
@@ -11,17 +11,18 @@ import (
|
||||
"crypto/md5"
|
||||
"strconv"
|
||||
|
||||
"os"
|
||||
"io"
|
||||
"log"
|
||||
"fmt"
|
||||
"errors"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"net/http"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -35,9 +36,9 @@ import (
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
gitProxy "github.com/go-git/go-git/v5/plumbing/transport"
|
||||
http2 "github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
|
||||
// Random
|
||||
xj "github.com/basgys/goxml2json"
|
||||
@@ -61,6 +62,7 @@ var registryName = "registry.hub.docker.com"
|
||||
var runningEnvironment = "onprem"
|
||||
|
||||
var syncUrl = "https://shuffler.io"
|
||||
//var syncUrl = "http://localhost:5002"
|
||||
|
||||
type retStruct struct {
|
||||
Success bool `json:"success"`
|
||||
@@ -447,7 +449,7 @@ func checkGitProxy(cloneOptions *git.CloneOptions) *git.CloneOptions {
|
||||
func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) error {
|
||||
// Returns false if there is an issue
|
||||
// Use this for register
|
||||
err := shuffle.CheckPasswordStrength(password)
|
||||
err := shuffle.CheckPasswordStrength(username, password)
|
||||
if err != nil {
|
||||
log.Printf("[WARNING] Bad password strength: %s", err)
|
||||
return err
|
||||
@@ -460,8 +462,6 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
//users, err := FindUser(ctx context.Context, username string) ([]User, error) {
|
||||
|
||||
users, err := shuffle.FindUser(ctx, strings.ToLower(strings.TrimSpace(username)))
|
||||
if err != nil && len(users) == 0 {
|
||||
log.Printf("[WARNING] Failed getting user %s: %s", username, err)
|
||||
@@ -486,7 +486,6 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini)
|
||||
newUser.Active = true
|
||||
newUser.Orgs = []string{org.Id}
|
||||
|
||||
// FIXME - Remove this later
|
||||
if role == "admin" {
|
||||
newUser.Role = "admin"
|
||||
newUser.Roles = []string{"admin"}
|
||||
@@ -1852,6 +1851,8 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
// return
|
||||
//}
|
||||
|
||||
log.Printf("[DEBUG] HOOKS: webhook callback: %s", request.URL.String())
|
||||
|
||||
if request.Method != "POST" {
|
||||
request.Method = "POST"
|
||||
}
|
||||
@@ -1863,6 +1864,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
|
||||
path := strings.Split(request.URL.String(), "/")
|
||||
if len(path) < 4 {
|
||||
log.Printf("[DEBUG] HOOKS: Invalid webhook path: %s", request.URL.String())
|
||||
resp.WriteHeader(403)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
@@ -1878,7 +1880,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
if location[1] == "api" {
|
||||
if len(location) <= 4 {
|
||||
log.Printf("[INFO] Couldn't handle location. Too short in webhook: %d", len(location))
|
||||
resp.WriteHeader(401)
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
}
|
||||
@@ -1895,6 +1897,8 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[DEBUG] HOOKS: Pre user agent check")
|
||||
|
||||
// Find user agent header
|
||||
userAgent := request.Header.Get("User-Agent")
|
||||
if strings.Contains(strings.ToLower(userAgent), "microsoftpreview") || strings.Contains(strings.ToLower(userAgent), "googlebot") {
|
||||
@@ -1917,8 +1921,8 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
//log.Printf("HookID: %s", hookId)
|
||||
hook, err := shuffle.GetHook(ctx, hookId)
|
||||
if err != nil {
|
||||
log.Printf("[WARNING] Failed getting hook %s (callback): %s", hookId, err)
|
||||
resp.WriteHeader(401)
|
||||
log.Printf("[WARNING] HOOKS: Failed getting hook %s (callback): %s", hookId, err)
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
}
|
||||
@@ -1930,21 +1934,21 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
//resp.WriteHeader(200)
|
||||
//resp.Write([]byte(`{"success": true}`))
|
||||
if hook.Status == "stopped" {
|
||||
log.Printf("[WARNING] Not running %s because hook status is stopped", hook.Id)
|
||||
resp.WriteHeader(401)
|
||||
log.Printf("[WARNING] HOOKS: Not running %s because hook status is stopped", hook.Id)
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Is it running?"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
if len(hook.Workflows) == 0 {
|
||||
log.Printf("[DEBUG] Not running because hook isn't connected to any workflows")
|
||||
resp.WriteHeader(401)
|
||||
log.Printf("[DEBUG] HOOKS: Not running because hook isn't connected to any workflows")
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflows are defined"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
if hook.Environment == "cloud" {
|
||||
log.Printf("[DEBUG] This should trigger in the cloud. Duplicate action allowed onprem.")
|
||||
log.Printf("[DEBUG] HOOKS: This should trigger in the cloud. Duplicate action allowed onprem.")
|
||||
}
|
||||
|
||||
// Check auth
|
||||
@@ -1960,7 +1964,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
|
||||
body, err := ioutil.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
log.Printf("[DEBUG] Body data error: %s", err)
|
||||
log.Printf("[DEBUG] HOOKS: data read error: %s", err)
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
@@ -2001,7 +2005,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
|
||||
b, err := json.Marshal(newBody)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed newBody marshaling for webhook: %s", err)
|
||||
log.Printf("[ERROR] HOOKS: Failed newBody marshaling for webhook: %s", err)
|
||||
resp.WriteHeader(500)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
@@ -2017,7 +2021,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
}
|
||||
|
||||
if len(hook.Start) == 0 {
|
||||
log.Printf("[WARNING] No start node for hook %s - running with workflow default.", hook.Id)
|
||||
log.Printf("[ERROR] HOOKS: No start node for hook %s - running with workflow default.", hook.Id)
|
||||
//bodyWrapper = string(parsedBody)
|
||||
}
|
||||
|
||||
@@ -2029,7 +2033,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
|
||||
// OrgId: activeOrgs[0].Id,
|
||||
workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest, hook.OrgId)
|
||||
|
||||
if err == nil {
|
||||
if hook.Version == "v2" {
|
||||
timeout := 15
|
||||
@@ -2064,6 +2067,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
} else {
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s"}`, workflowExecution.ExecutionId)))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -2071,6 +2075,10 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp)))
|
||||
}
|
||||
|
||||
log.Printf("[ERROR] HOOKS: END OF FUNCTION FOR '%s'. IF this is reached, something went wrong.", hook.Id)
|
||||
resp.WriteHeader(500)
|
||||
resp.Write([]byte(`{"success": false, "reason": "Failed to run workflow. Check logs."}`))
|
||||
|
||||
}
|
||||
|
||||
func handlePipelineCallback(resp http.ResponseWriter, request *http.Request) {
|
||||
@@ -3087,7 +3095,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s
|
||||
return
|
||||
}
|
||||
|
||||
if user.Id == app.Owner || (user.Role == "admin" && user.ActiveOrg.Id == app.ReferenceOrg) || shuffle.ArrayContains(app.Contributors, user.Id) {
|
||||
if user.Id == app.Owner || (user.Role == "admin" && user.ActiveOrg.Id == app.ReferenceOrg) || shuffle.ArrayContains(app.Contributors, user.Id) {
|
||||
log.Printf("[DEBUG] Editing app %s with user %s (%s) in org %s", test.Id, user.Username, user.Id, user.ActiveOrg.Id)
|
||||
} else {
|
||||
log.Printf("[WARNING] Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name)
|
||||
@@ -3375,7 +3383,6 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User, s
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
log.Printf("[DEBUG] Successfully built app %s (%s)", api.Name, api.ID)
|
||||
if len(user.Id) > 0 {
|
||||
resp.WriteHeader(200)
|
||||
@@ -3799,30 +3806,56 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error {
|
||||
}
|
||||
}
|
||||
|
||||
if org.SyncConfig.WorkflowBackup {
|
||||
workflows, err := shuffle.GetAllWorkflowsByQuery(ctx, foundUser, 250, "")
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting backup workflows for org %s: %s", org.Id, err)
|
||||
} else {
|
||||
backupJob.Workflows = workflows
|
||||
}
|
||||
// Check if it's 1/20 times (600 seconds - 10 min on average)
|
||||
// Only problem: May take time to sync the first time, which is annoying
|
||||
shouldBackupData := false
|
||||
randomNumber := rand.Intn(20)
|
||||
if randomNumber == 0 {
|
||||
shouldBackupData = true
|
||||
}
|
||||
|
||||
if org.SyncConfig.AppBackup && len(org.Users) > 0 {
|
||||
|
||||
apps, err := shuffle.GetPrioritizedApps(ctx, foundUser)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting backup apps for org %s: %s", org.Id, err)
|
||||
} else {
|
||||
backupJob.Apps = apps
|
||||
// Just to prevent it from spamming large outbound requests
|
||||
if shouldBackupData {
|
||||
if org.SyncConfig.WorkflowBackup {
|
||||
workflows, err := shuffle.GetAllWorkflowsByQuery(ctx, foundUser, 250, "")
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting backup workflows for org %s: %s", org.Id, err)
|
||||
} else {
|
||||
backupJob.Workflows = workflows
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
info, err := shuffle.GetOrgStatistics(ctx, org.Id)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err)
|
||||
} else {
|
||||
backupJob.Stats = *info
|
||||
if org.SyncConfig.AppBackup && len(org.Users) > 0 {
|
||||
foundUser.ActiveOrg.Id = org.Id
|
||||
apps, err := shuffle.GetPrioritizedApps(ctx, foundUser)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting backup apps for org %s: %s", org.Id, err)
|
||||
} else {
|
||||
parsedApps := []shuffle.WorkflowApp{}
|
||||
for _, app := range apps {
|
||||
if len(app.Actions) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if !app.Generated {
|
||||
continue
|
||||
}
|
||||
|
||||
parsedApps = append(parsedApps, app)
|
||||
}
|
||||
|
||||
backupJob.Apps = parsedApps
|
||||
}
|
||||
}
|
||||
|
||||
// Send stats once every 10 times or so..?
|
||||
// For now, just send every time
|
||||
info, err := shuffle.GetOrgStatistics(ctx, org.Id)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed getting org statistics backup for org %s: %s", org.Id, err)
|
||||
} else {
|
||||
backupJob.Stats = *info
|
||||
}
|
||||
}
|
||||
|
||||
backupJobData, err := json.Marshal(backupJob)
|
||||
@@ -3859,6 +3892,7 @@ func remoteOrgJobHandler(org shuffle.Org, interval int) error {
|
||||
//log.Printf("[ERROR] Failed cloud sync job controller run for '%s': %s", respBody, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3996,6 +4030,8 @@ func runInitEs(ctx context.Context) {
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
|
||||
// FIXME: This should ONLY run on one backend instance
|
||||
|
||||
schedules, err := shuffle.GetAllSchedules(ctx, "ALL")
|
||||
if err != nil {
|
||||
log.Printf("[WARNING] Failed getting schedules during service init: %s", err)
|
||||
@@ -4139,7 +4175,7 @@ func runInitEs(ctx context.Context) {
|
||||
}
|
||||
|
||||
//interval := int(org.SyncConfig.Interval)
|
||||
interval := 15
|
||||
interval := 30
|
||||
if interval == 0 {
|
||||
log.Printf("[WARNING] Skipping org %s because sync isn't set (0).", org.Id)
|
||||
continue
|
||||
@@ -4241,17 +4277,20 @@ func runInitEs(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
|
||||
if newresp.StatusCode != 200 {
|
||||
log.Printf("[WARNING] Failed stopping runs in environment %s. Status code: %d", environment, newresp.StatusCode)
|
||||
|
||||
respBody, err := ioutil.ReadAll(newresp.Body)
|
||||
if err != nil {
|
||||
log.Printf("[ERROR] Failed setting respbody %s for execution stop. Status: %d", err, newresp.StatusCode)
|
||||
continue
|
||||
}
|
||||
|
||||
//respBody, err := ioutil.ReadAll(newresp.Body)
|
||||
//if err != nil {
|
||||
// log.Printf("[ERROR] Failed setting respbody %s", err)
|
||||
// continue
|
||||
//}
|
||||
//log.Printf("[DEBUG] Successfully ran workflow cleanup request for %s. Body: %s", environment, string(respBody))
|
||||
if newresp.StatusCode != 200 {
|
||||
if !strings.Contains(string(respBody), "is active") {
|
||||
log.Printf("[WARNING] Failed stopping runs in environment %s. Status code: %d. Body: %s", environment, newresp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
url = fmt.Sprintf("http://localhost:%s/api/v1/environments/%s/rerun", backendPort, environment)
|
||||
req, err = http.NewRequest(
|
||||
@@ -4377,7 +4416,7 @@ func runInitEs(ctx context.Context) {
|
||||
}
|
||||
|
||||
if os.Getenv("SHUFFLE_HEALTHCHECK_DISABLED") != "true" {
|
||||
healthcheckInterval := 60
|
||||
healthcheckInterval := 60
|
||||
log.Printf("[INFO] Starting healthcheck job every %d minute. Stats available on /api/v1/health/stats, and dashboard on /health. Disable with SHUFFLE_HEALTHCHECK_DISABLED=true", healthcheckInterval)
|
||||
job := func() {
|
||||
// Prepare a fake http.responsewriter
|
||||
@@ -4669,7 +4708,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
|
||||
// If you want to disable cloud sync, see previous section.
|
||||
if org.CloudSync {
|
||||
log.Printf("[WARNING] Org %s is already syncing. Skip", org.Id)
|
||||
resp.WriteHeader(401)
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Your org is already syncing. Nothing to set up."}`)))
|
||||
return
|
||||
}
|
||||
@@ -4746,6 +4785,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
|
||||
org.SyncConfig = shuffle.SyncConfig{
|
||||
Apikey: responseData.SessionKey,
|
||||
Interval: responseData.IntervalSeconds,
|
||||
|
||||
WorkflowBackup: true,
|
||||
AppBackup: true,
|
||||
}
|
||||
|
||||
interval := int(responseData.IntervalSeconds)
|
||||
|
||||
@@ -1905,8 +1905,8 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) {
|
||||
|
||||
err = shuffle.SetSchedule(ctx, newSchedule)
|
||||
if err != nil {
|
||||
log.Printf("Failed setting cloud schedule: %s", err)
|
||||
resp.WriteHeader(401)
|
||||
log.Printf("[ERROR] Failed setting cloud schedule: %s", err)
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
|
||||
return
|
||||
}
|
||||
@@ -1941,17 +1941,22 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) {
|
||||
|
||||
// FIXME - real error message lol
|
||||
if err != nil {
|
||||
log.Printf("Failed creating schedule: %s", err)
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Invalid argument. Try cron */15 * * * *"}`)))
|
||||
log.Printf("[ERROR] Failed creating schedule: %s", err)
|
||||
|
||||
resp.WriteHeader(400)
|
||||
if schedule.Environment == "cloud" {
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Invalid argument. For cloud schedules, try cron */15 * * * *"}`)))
|
||||
} else {
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Invalid argument. For onprem schedules, try 60 for 60 seconds"}`)))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
//workflow.Schedules = append(workflow.Schedules, schedule)
|
||||
err = shuffle.SetWorkflow(ctx, *workflow, workflow.ID)
|
||||
if err != nil {
|
||||
log.Printf("Failed setting workflow for schedule: %s", err)
|
||||
resp.WriteHeader(401)
|
||||
log.Printf("[ERROR] Failed setting workflow for schedule: %s", err)
|
||||
resp.WriteHeader(400)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user