hotfix: fixing image errors
This commit is contained in:
@@ -3,7 +3,6 @@ package main
|
|||||||
import (
|
import (
|
||||||
"github.com/shuffle/shuffle-shared"
|
"github.com/shuffle/shuffle-shared"
|
||||||
|
|
||||||
|
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -22,8 +21,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/docker/docker/api/types"
|
"github.com/docker/docker/api/types"
|
||||||
"github.com/docker/docker/api/types/filters"
|
|
||||||
"github.com/docker/docker/api/types/container"
|
"github.com/docker/docker/api/types/container"
|
||||||
|
"github.com/docker/docker/api/types/filters"
|
||||||
"github.com/docker/docker/api/types/mount"
|
"github.com/docker/docker/api/types/mount"
|
||||||
dockerclient "github.com/docker/docker/client"
|
dockerclient "github.com/docker/docker/client"
|
||||||
// This is for automatic removal of certain code :)
|
// This is for automatic removal of certain code :)
|
||||||
@@ -52,9 +51,7 @@ var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP"))
|
|||||||
var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME")
|
var swarmNetworkName = os.Getenv("SHUFFLE_SWARM_NETWORK_NAME")
|
||||||
var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION"))
|
var dockerApiVersion = strings.ToLower(os.Getenv("DOCKER_API_VERSION"))
|
||||||
|
|
||||||
//var baseimagename = "frikky/shuffle"
|
var baseimagename = "frikky/shuffle"
|
||||||
var baseimagerepository = os.Getenv("SHUFFLE_BASE_IMAGE_REPOSITORY")
|
|
||||||
var baseimagename = fmt.Sprintf("%s/%s", baseimagerepository, os.Getenv("SHUFFLE_BASE_IMAGE_NAME"))
|
|
||||||
|
|
||||||
// var baseimagename = "registry.hub.docker.com/frikky/shuffle"
|
// var baseimagename = "registry.hub.docker.com/frikky/shuffle"
|
||||||
var registryName = "registry.hub.docker.com"
|
var registryName = "registry.hub.docker.com"
|
||||||
@@ -80,6 +77,7 @@ var startAction string
|
|||||||
//var allLogs map[string]string
|
//var allLogs map[string]string
|
||||||
//var containerIds []string
|
//var containerIds []string
|
||||||
var downloadedImages []string
|
var downloadedImages []string
|
||||||
|
|
||||||
type ImageDownloadBody struct {
|
type ImageDownloadBody struct {
|
||||||
Image string `json:"image"`
|
Image string `json:"image"`
|
||||||
}
|
}
|
||||||
@@ -90,7 +88,6 @@ type ImageRequest struct {
|
|||||||
|
|
||||||
var finishedExecutions []string
|
var finishedExecutions []string
|
||||||
|
|
||||||
|
|
||||||
// Images to be autodeployed in the latest version of Shuffle.
|
// Images to be autodeployed in the latest version of Shuffle.
|
||||||
var autoDeploy = map[string]string{
|
var autoDeploy = map[string]string{
|
||||||
"http:1.4.0": "frikky/shuffle:http_1.4.0",
|
"http:1.4.0": "frikky/shuffle:http_1.4.0",
|
||||||
@@ -137,7 +134,6 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
handleExecutionResult(workflowExecution)
|
handleExecutionResult(workflowExecution)
|
||||||
validated := shuffle.ValidateFinished(ctx, -1, workflowExecution)
|
validated := shuffle.ValidateFinished(ctx, -1, workflowExecution)
|
||||||
if validated {
|
if validated {
|
||||||
@@ -190,14 +186,12 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for _, trigger := range workflowExecution.Workflow.Triggers {
|
for _, trigger := range workflowExecution.Workflow.Triggers {
|
||||||
//log.Printf("[DEBUG] Found trigger %s", trigger.AppName)
|
//log.Printf("[DEBUG] Found trigger %s", trigger.AppName)
|
||||||
if trigger.AppName != "User Input" && trigger.AppName != "Shuffle Workflow" && trigger.AppName != "shuffle-subflow" {
|
if trigger.AppName != "User Input" && trigger.AppName != "Shuffle Workflow" && trigger.AppName != "shuffle-subflow" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// check if it has wait for results in params
|
// check if it has wait for results in params
|
||||||
wait := false
|
wait := false
|
||||||
for _, param := range trigger.Parameters {
|
for _, param := range trigger.Parameters {
|
||||||
@@ -251,7 +245,6 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo
|
|||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -273,7 +266,6 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// removes every container except itself (worker)
|
// removes every container except itself (worker)
|
||||||
func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) {
|
func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) {
|
||||||
log.Printf("[DEBUG][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v, Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend, workflowExecution.ExecutionParent)
|
log.Printf("[DEBUG][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v, Parent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend, workflowExecution.ExecutionParent)
|
||||||
@@ -494,7 +486,6 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Max 10% CPU every second
|
// Max 10% CPU every second
|
||||||
//CPUShares: 128,
|
//CPUShares: 128,
|
||||||
//CPUQuota: 10000,
|
//CPUQuota: 10000,
|
||||||
@@ -562,7 +553,6 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
|
|||||||
Env: env,
|
Env: env,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Checking as late as possible, just in case.
|
// Checking as late as possible, just in case.
|
||||||
newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID)
|
newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID)
|
||||||
_, err := shuffle.GetCache(ctx, newExecId)
|
_, err := shuffle.GetCache(ctx, newExecId)
|
||||||
@@ -901,7 +891,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId)
|
startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId)
|
||||||
|
|
||||||
dockercli, err := dockerclient.NewEnvClient()
|
dockercli, err := dockerclient.NewEnvClient()
|
||||||
@@ -1057,7 +1046,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
|
|||||||
env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")))
|
env = append(env, fmt.Sprintf("SHUFFLE_APP_SDK_TIMEOUT=%s", os.Getenv("SHUFFLE_APP_SDK_TIMEOUT")))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Fixes issue:
|
// Fixes issue:
|
||||||
// standard_go init_linux.go:185: exec user process caused "argument list too long"
|
// standard_go init_linux.go:185: exec user process caused "argument list too long"
|
||||||
// https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083
|
// https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083
|
||||||
@@ -1085,8 +1073,6 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
|
|||||||
fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion),
|
fmt.Sprintf("%s:%s_%s", baseimagename, parsedAppname, action.AppVersion),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// If cleanup is set, it should run for efficiency
|
// If cleanup is set, it should run for efficiency
|
||||||
pullOptions := types.ImagePullOptions{}
|
pullOptions := types.ImagePullOptions{}
|
||||||
if cleanupEnv == "true" {
|
if cleanupEnv == "true" {
|
||||||
@@ -1980,7 +1966,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
|
|||||||
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Execution is not executing, but %s"}`, workflowExecution.Status)))
|
resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Execution is not executing, but %s"}`, workflowExecution.Status)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
log.Printf("[DEBUG][%s] Shutting down (35)", workflowExecution.ExecutionId)
|
log.Printf("[DEBUG][%s] Shutting down (35)", workflowExecution.ExecutionId)
|
||||||
|
|
||||||
// Force sending result
|
// Force sending result
|
||||||
@@ -2096,7 +2081,6 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
|
|||||||
|
|
||||||
func sendSelfRequest(actionResult shuffle.ActionResult) {
|
func sendSelfRequest(actionResult shuffle.ActionResult) {
|
||||||
|
|
||||||
|
|
||||||
data, err := json.Marshal(actionResult)
|
data, err := json.Marshal(actionResult)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[ERROR][%s] Shutting down (24): Failed to unmarshal data for backend: %s", actionResult.ExecutionId, err)
|
log.Printf("[ERROR][%s] Shutting down (24): Failed to unmarshal data for backend: %s", actionResult.ExecutionId, err)
|
||||||
@@ -2233,7 +2217,6 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
log.Printf("[DEBUG][%s] Should send full result to %s", workflowExecution.ExecutionId, baseUrl)
|
log.Printf("[DEBUG][%s] Should send full result to %s", workflowExecution.ExecutionId, baseUrl)
|
||||||
|
|
||||||
//data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization)
|
//data = fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization)
|
||||||
@@ -2317,7 +2300,6 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) {
|
|||||||
// GetLocalIP returns the non loopback local IP of the host
|
// GetLocalIP returns the non loopback local IP of the host
|
||||||
func getLocalIP() string {
|
func getLocalIP() string {
|
||||||
|
|
||||||
|
|
||||||
addrs, err := net.InterfaceAddrs()
|
addrs, err := net.InterfaceAddrs()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ""
|
return ""
|
||||||
@@ -2363,7 +2345,6 @@ func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener {
|
|||||||
|
|
||||||
log.Printf("[DEBUG] OLD HOSTNAME: %s", appCallbackUrl)
|
log.Printf("[DEBUG] OLD HOSTNAME: %s", appCallbackUrl)
|
||||||
|
|
||||||
|
|
||||||
port := listener.Addr().(*net.TCPAddr).Port
|
port := listener.Addr().(*net.TCPAddr).Port
|
||||||
// Set the port environment variable
|
// Set the port environment variable
|
||||||
os.Setenv("WORKER_PORT", fmt.Sprintf("%d", port))
|
os.Setenv("WORKER_PORT", fmt.Sprintf("%d", port))
|
||||||
@@ -2521,7 +2502,6 @@ func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) {
|
|||||||
*/
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Runs data discovery
|
// Runs data discovery
|
||||||
|
|
||||||
func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error {
|
func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error {
|
||||||
@@ -2848,7 +2828,6 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe
|
|||||||
|
|
||||||
// Set environment variable
|
// Set environment variable
|
||||||
|
|
||||||
|
|
||||||
//log.Printf("Before wait")
|
//log.Printf("Before wait")
|
||||||
//wg := sync.WaitGroup{}
|
//wg := sync.WaitGroup{}
|
||||||
//wg.Add(1)
|
//wg.Add(1)
|
||||||
@@ -2910,7 +2889,7 @@ func main() {
|
|||||||
timezone = "Europe/Amsterdam"
|
timezone = "Europe/Amsterdam"
|
||||||
}
|
}
|
||||||
|
|
||||||
if baseimagename == "" || baseimagename == "/" {
|
if baseimagename == "" {
|
||||||
log.Printf("[DEBUG] Setting baseimagename")
|
log.Printf("[DEBUG] Setting baseimagename")
|
||||||
baseimagename = "frikky/shuffle" // Dockerhub
|
baseimagename = "frikky/shuffle" // Dockerhub
|
||||||
//baseimagename = "shuffle" // Github (ghcr.io)
|
//baseimagename = "shuffle" // Github (ghcr.io)
|
||||||
@@ -2920,7 +2899,6 @@ func main() {
|
|||||||
swarmConfig := os.Getenv("SHUFFLE_SWARM_CONFIG")
|
swarmConfig := os.Getenv("SHUFFLE_SWARM_CONFIG")
|
||||||
log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, swarmConfig)
|
log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, swarmConfig)
|
||||||
|
|
||||||
|
|
||||||
authorization := ""
|
authorization := ""
|
||||||
executionId := ""
|
executionId := ""
|
||||||
|
|
||||||
@@ -3225,7 +3203,6 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
for _, img := range images {
|
for _, img := range images {
|
||||||
for _, tag := range img.RepoTags {
|
for _, tag := range img.RepoTags {
|
||||||
splitTag := strings.Split(tag, ":")
|
splitTag := strings.Split(tag, ":")
|
||||||
@@ -3238,7 +3215,7 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) {
|
|||||||
possibleNames = append(possibleNames, fmt.Sprintf("frikky/shuffle:%s", baseTag))
|
possibleNames = append(possibleNames, fmt.Sprintf("frikky/shuffle:%s", baseTag))
|
||||||
possibleNames = append(possibleNames, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", baseTag))
|
possibleNames = append(possibleNames, fmt.Sprintf("registry.hub.docker.com/frikky/shuffle:%s", baseTag))
|
||||||
|
|
||||||
if (arrayContains(possibleNames, image.Image)) {
|
if arrayContains(possibleNames, image.Image) {
|
||||||
log.Printf("[DEBUG] Image %s already downloaded that has been requested to download", image.Image)
|
log.Printf("[DEBUG] Image %s already downloaded that has been requested to download", image.Image)
|
||||||
resp.WriteHeader(200)
|
resp.WriteHeader(200)
|
||||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "image already present"}`)))
|
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "image already present"}`)))
|
||||||
@@ -3263,7 +3240,6 @@ func runWebserver(listener net.Listener) {
|
|||||||
r.HandleFunc("/api/v1/run", handleRunExecution).Methods("POST", "OPTIONS")
|
r.HandleFunc("/api/v1/run", handleRunExecution).Methods("POST", "OPTIONS")
|
||||||
r.HandleFunc("/api/v1/download", handleDownloadImage).Methods("POST", "OPTIONS")
|
r.HandleFunc("/api/v1/download", handleDownloadImage).Methods("POST", "OPTIONS")
|
||||||
|
|
||||||
|
|
||||||
if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" {
|
if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" {
|
||||||
r.HandleFunc("/debug/pprof/", pprof.Index)
|
r.HandleFunc("/debug/pprof/", pprof.Index)
|
||||||
r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
|
r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
|
||||||
|
|||||||
Reference in New Issue
Block a user