marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed network/hostname matching issue with wrong network interface

Browse Source
This commit is contained in:
frikky
2021-12-19 01:14:51 +01:00
parent 1115a8cade
commit f09e00402c
9 changed files with 217 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/go.mod
+1 -1
View File
@@ -22,7 +22,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/h2non/filetype v1.1.1
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.1.58
github.com/shuffle/shuffle-shared v0.1.60
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
frontend/src/views/AppCreator.jsx
+90 -37
View File
@@ -1718,6 +1718,10 @@ const AppCreator = (props) => {
},
};
if (queryitem.example !== undefined) {
newitem.example = queryitem.example
}
if (queryitem.description !== undefined) {
newitem.description = queryitem.description;
}
@@ -2129,7 +2133,7 @@ const AppCreator = (props) => {
};
const addPathQuery = () => {
urlPathQueries.push({ name: "", required: true });
urlPathQueries.push({ name: "", required: true, example: "", });
if (updater === "addupdater") {
setUpdater("updater");
} else {
@@ -2613,6 +2617,55 @@ const AppCreator = (props) => {
return (
<Paper key={index} style={actionListStyle}>
<div style={{ marginLeft: "5px", width: "100%" }}>
<div style={{display: "flex"}}>
<TextField
required
fullWidth={true}
defaultValue={data.name}
placeholder={"Query name (key)"}
label={"Query Key"}
helperText={
<span style={{ color: "white", marginBottom: "2px" }}>
Click required to flip
</span>
}
onBlur={(e) => {
console.log("IN BLUR: ", e.target.value);
urlPathQueries[index].name = e.target.value.replaceAll(
"=",
""
);
setUrlPathQueries(urlPathQueries);
}}
style={{flex: 3}}
InputProps={{
style: {
color: "white",
},
}}
/>
<TextField
fullWidth={true}
defaultValue={data.example}
placeholder={"Default value"}
label={"Example"}
onBlur={(e) => {
urlPathQueries[index].example = e.target.value.replaceAll(
"=",
""
)
setUrlPathQueries(urlPathQueries)
}}
style={{flex: 2}}
InputProps={{
style: {
color: "white",
},
}}
/>
</div>
<div
style={{ cursor: "pointer" }}
onClick={() => {
@@ -2624,31 +2677,6 @@ const AppCreator = (props) => {
{data.required.toString()}
</div>
</div>
<TextField
required
fullWidth={true}
defaultValue={data.name}
placeholder={"Query name"}
helperText={
<span style={{ color: "white", marginBottom: "2px" }}>
Click required switch
</span>
}
onBlur={(e) => {
console.log("IN BLUR: ", e.target.value);
urlPathQueries[index].name = e.target.value.replaceAll(
"=",
""
);
setUrlPathQueries(urlPathQueries);
}}
InputProps={{
style: {
color: "white",
},
}}
/>
</div>
<div
style={{ float: "right", color: "#f85a3e", cursor: "pointer" }}
@@ -2656,7 +2684,7 @@ const AppCreator = (props) => {
deletePathQuery(index);
}}
>
Delete
<DeleteIcon />
</div>
</Paper>
);
@@ -3451,7 +3479,26 @@ const AppCreator = (props) => {
}
if (parsedurl.includes("?")) {
parsedurl = parsedurl.split("?")[0]
const parsedurlsplit = parsedurl.split("?")
parsedurl = parsedurlsplit[0]
//var newqueries = selectedAction.queries === undefined || selectedAction.queries === null ? [] : selectedAction.queries
const datasplit = parsedurlsplit[1].split("&")
for (var key in datasplit) {
console.log("Data: ", datasplit[key])
var actualkey = datasplit[key]
var example = ""
if (datasplit[key].includes("=")) {
actualkey = datasplit[key].split("=")[0]
example = datasplit[key].split("=")[1]
}
const foundPath = urlPathQueries.find(data => data.name === actualkey)
if (foundPath === null || foundPath === undefined) {
urlPathQueries.push({ name: actualkey, example: example, required: true })
}
}
}
if (event.target.value !== parsedurl) {
@@ -3663,15 +3710,21 @@ const AppCreator = (props) => {
}
style={{ backgroundColor: inputColor, color: "white", height: "50px" }}
>
{categories.map((data, index) => (
<MenuItem
key={index}
style={{ backgroundColor: inputColor, color: "white" }}
value={data}
>
{data}
</MenuItem>
))}
{categories.map((data, index) => {
if (data === undefined || data === null || data === "") {
return null
}
return (
<MenuItem
key={index}
style={{ backgroundColor: inputColor, color: "white" }}
value={data}
>
{data}
</MenuItem>
)
})}
</Select>
<h4>Tags</h4>
<ChipInput
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.9.35
VERSION=0.9.42
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+13 -9
View File
@@ -250,7 +250,7 @@ func deployServiceWorkers(image string) {
log.Printf("[DEBUG] Found %d node(s) to replicate over. Defaulting to 1 IF we can't auto-discover them.", cnt)
replicatedJobs := uint64(replicas * nodeCount)
log.Printf("[DEBUG] Deploying %d containers for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
log.Printf("[DEBUG] Deploying %d container(s) for worker with swarm to each node. Service name: %s. Image: %s", replicas, innerContainerName, image)
if timezone == "" {
timezone = "Europe/Amsterdam"
@@ -332,7 +332,7 @@ func deployServiceWorkers(image string) {
)
if err == nil {
log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d nodes", replicas, cnt)
log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt)
//time.Sleep(time.Duration(10) * time.Second)
//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
} else {
@@ -393,7 +393,6 @@ func deployWorker(image string, identifier string, env []string, executionReques
}
if err == nil {
log.Printf("[DEBUG] Started worker from request with name: %s", executionRequest.ExecutionId)
executionIds = append(executionIds, executionRequest.ExecutionId)
}
}()
@@ -727,20 +726,20 @@ func main() {
)
if err != nil {
log.Printf("[ERROR] Failed making request builder: %s", err)
log.Printf("[ERROR] Failed making request builder during init: %s", err)
os.Exit(3)
}
zombiecounter := 0
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Org-Id", orgId)
log.Printf("[INFO] Waiting for executions at %s", fullUrl)
log.Printf("[INFO] Waiting for executions at %s with Org ID %s", fullUrl, orgId)
hasStarted := false
for {
//log.Printf("Prerequest")
//go getStats()
newresp, err := client.Do(req)
//log.Printf("Prerequest")
//log.Printf("Postrequest")
newresp, err := client.Do(req)
if err != nil {
log.Printf("[WARNING] Failed making request: %s", err)
zombiecounter += 1
@@ -758,6 +757,10 @@ func main() {
log.Printf("[WARNING] Bad statuscode: %d", newresp.StatusCode)
}
} else {
if !hasStarted {
log.Printf("[DEBUG] Starting iteration. Got statuscode %d from backend on first request", newresp.StatusCode)
}
hasStarted = true
}
@@ -1160,10 +1163,11 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest) error {
body, err := ioutil.ReadAll(newresp.Body)
if err != nil {
log.Printf("[ERROR] Failed reading body in worker request: %s", err)
log.Printf("[ERROR] Failed reading body in worker request body: %s", err)
return err
}
_ = body
log.Printf("[DEBUG] NEWRESP (from worker request %s): %s (Status: %d)", workflowExecution.ExecutionId, string(body), newresp.StatusCode)
log.Printf("[DEBUG] Ran worker from request with execution ID: %s. Worker URL: %s.\n\n DEBUGGING: docker service logs shuffle-workers | grep %s\n\n", workflowExecution.ExecutionId, streamUrl, workflowExecution.ExecutionId)
return nil
}
functions/onprem/worker/Dockerfile
+1
View File
@@ -6,6 +6,7 @@ WORKDIR /app
#RUN go env -w GO111MODULE=auto
COPY worker.go /app/worker.go
COPY go.mod /app/go.mod
#COPY go.sum /app/go.sum
#RUN go
#COPY go.sum /app/go.sum
RUN go get
functions/onprem/worker/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.9.40
VERSION=0.9.42
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
functions/onprem/worker/go.mod
+1 -1
View File
@@ -10,6 +10,6 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/shuffle/shuffle-shared v0.1.55
github.com/shuffle/shuffle-shared v0.1.60
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
)
functions/onprem/worker/go.sum
+2
View File
@@ -578,6 +578,8 @@ github.com/shuffle/shuffle-shared v0.1.54 h1:dHpwot+5RPX8k9EC/8Yd+QYsFqCsqsv+J1w
github.com/shuffle/shuffle-shared v0.1.54/go.mod h1:2ndjLm4ZOvY6arGFwOgGnkQ457Ke7gka9HDF/EkdIxQ=
github.com/shuffle/shuffle-shared v0.1.55 h1:feHtTN7Uhr1aMxkMIo3xZbr97599VB2eLekogTj/9Z4=
github.com/shuffle/shuffle-shared v0.1.55/go.mod h1:2ndjLm4ZOvY6arGFwOgGnkQ457Ke7gka9HDF/EkdIxQ=
github.com/shuffle/shuffle-shared v0.1.60 h1:Jjb6TfE/KnVfCryIL2vtRHBnBX307slVlGBjaEqgwW4=
github.com/shuffle/shuffle-shared v0.1.60/go.mod h1:2ndjLm4ZOvY6arGFwOgGnkQ457Ke7gka9HDF/EkdIxQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
functions/onprem/worker/worker.go
+107 -45
View File
@@ -90,7 +90,7 @@ type UserInputSubflow struct {
// removes every container except itself (worker)
func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) {
log.Printf("[INFO] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v", workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend)
log.Printf("[INFO][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend)
//reason := "Error in execution"
sleepDuration := 1
@@ -98,9 +98,9 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
shutdownData, err := json.Marshal(workflowExecution)
if err == nil {
sendResult(workflowExecution, shutdownData)
log.Printf("[WARNING] Sent shutdown update with %d results and result value %s", len(workflowExecution.Results), reason)
log.Printf("[WARNING][%s] Sent shutdown update with %d results and result value %s", workflowExecution.ExecutionId, len(workflowExecution.Results), reason)
} else {
log.Printf("[WARNING] Failed to send update: %s", err)
log.Printf("[WARNING][%s] Failed to send update: %s", workflowExecution.ExecutionId, err)
}
time.Sleep(time.Duration(sleepDuration) * time.Second)
@@ -127,7 +127,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
}
*/
} else {
log.Printf("[DEBUG] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv)
log.Printf("[DEBUG][%s] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", workflowExecution.ExecutionId, len(containerIds), cleanupEnv)
}
@@ -145,7 +145,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
//fmt.Println(url.QueryEscape(query))
abortUrl += path
log.Printf("[DEBUG] Abort URL: %s", abortUrl)
log.Printf("[DEBUG][%s] Abort URL: %s", workflowExecution.ExecutionId, abortUrl)
req, err := http.NewRequest(
"GET",
@@ -154,7 +154,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
)
if err != nil {
log.Println("[INFO] Failed building request: %s", err)
log.Println("[INFO][%s] Failed building request: %s", workflowExecution.ExecutionId, err)
}
// FIXME: Add an API call to the backend
@@ -163,7 +163,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
if len(authorization) > 0 {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authorization))
} else {
log.Printf("[ERROR] No authorization specified for abort")
log.Printf("[ERROR][%s] No authorization specified for abort", workflowExecution.ExecutionId)
}
} else {
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", workflowExecution.Authorization))
@@ -182,23 +182,23 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
log.Printf("[INFO][%s] Running with HTTP proxy %s (env: HTTP_PROXY)", workflowExecution.ExecutionId, httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
log.Printf("[INFO][%s] Running with HTTPS proxy %s (env: HTTPS_PROXY)", workflowExecution.ExecutionId, httpsProxy)
}
}
log.Printf("[DEBUG] All App Logs: %#v", allLogs)
log.Printf("[DEBUG][%s] All App Logs: %#v", workflowExecution.ExecutionId, allLogs)
_, err = client.Do(req)
if err != nil {
log.Printf("[WARNING] Failed abort request: %s", err)
log.Printf("[WARNING][%s] Failed abort request: %s", workflowExecution.ExecutionId, err)
}
} else {
log.Printf("[INFO] NOT running abort during shutdown.")
log.Printf("[INFO][%s] NOT running abort during shutdown.", workflowExecution.ExecutionId)
}
log.Printf("[INFO] Finished shutdown (after %d seconds). ", sleepDuration)
log.Printf("[INFO][%s] Finished shutdown (after %d seconds). ", workflowExecution.ExecutionId, sleepDuration)
//Finished shutdown (after %d seconds). ", sleepDuration)
// Allows everything to finish in subprocesses (apps)
@@ -206,7 +206,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
time.Sleep(time.Duration(sleepDuration) * time.Second)
os.Exit(3)
} else {
log.Printf("\n\n[DEBUG] Sending result and resetting values (K8s & Swarm).\n\n")
log.Printf("\n\n[DEBUG][%s] Sending result and resetting values (K8s & Swarm).\n\n", workflowExecution.ExecutionId)
//UpdateExecutionVariables(ctx, workflowExecution.ExecutionId, startAction, children, parents, visited, executed, nextActions, environments, extra)
/*
@@ -239,7 +239,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
appName := strings.Replace(identifier, fmt.Sprintf("_%s", action.ID), "", -1)
appName = strings.Replace(appName, fmt.Sprintf("_%s", workflowExecution.ExecutionId), "", -1)
appName = strings.ToLower(appName)
log.Printf("[INFO] New appname: %s, image: %s", appName, image)
log.Printf("[INFO][%s] New appname: %s, image: %s", workflowExecution.ExecutionId, appName, image)
if !shuffle.ArrayContains(downloadedImages, image) {
log.Printf("[DEBUG] Downloading image %s from backend as it's first iteration for this image on the worker.", image)
@@ -261,7 +261,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
return err
}
log.Printf("[DEBUG] Should run towards port %d for app %s", exposedPort, appName)
log.Printf("[DEBUG][%s] Should run towards port %d for app %s", workflowExecution.ExecutionId, exposedPort, appName)
err = sendAppRequest(baseUrl, appName, exposedPort, action, workflowExecution)
if err != nil {
log.Printf("[ERROR] Failed sending request to app %s on port %d: %s", appName, exposedPort, err)
@@ -647,14 +647,14 @@ func removeIndex(s []string, i int) []string {
func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
ctx := context.Background()
startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId)
log.Printf("[DEBUG] Getting info for %s. Extra: %d", workflowExecution.ExecutionId, extra)
log.Printf("[DEBUG][%s] Getting info for %s. Extra: %d", workflowExecution.ExecutionId, workflowExecution.ExecutionId, extra)
dockercli, err := dockerclient.NewEnvClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (3): %s", err)
return
}
log.Printf("[INFO] Inside execution results with %d / %d results", len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra)
log.Printf("[INFO][%s] Inside execution results with %d / %d results", workflowExecution.ExecutionId, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions)+extra)
if len(startAction) == 0 {
startAction = workflowExecution.Start
@@ -704,16 +704,16 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
if isSkipped {
//log.Printf("Skipping %s as all parents are done", item.Action.Label)
if !arrayContains(visited, item.Action.ID) {
log.Printf("[INFO] Adding visited (1): %s\n", item.Action.Label)
log.Printf("[INFO][%s] Adding visited (1): %s\n", workflowExecution.ExecutionId, item.Action.Label)
visited = append(visited, item.Action.ID)
}
} else {
log.Printf("[INFO] Continuing %s as all parents are NOT done", item.Action.Label)
log.Printf("[INFO][%s] Continuing %s as all parents are NOT done", workflowExecution.ExecutionId, item.Action.Label)
appendActions = append(appendActions, item.Action.ID)
}
} else {
if item.Status == "FINISHED" {
log.Printf("[INFO] Adding visited (2): %s\n", item.Action.Label)
log.Printf("[INFO][%s] Adding visited (2): %s\n", workflowExecution.ExecutionId, item.Action.Label)
visited = append(visited, item.Action.ID)
}
}
@@ -843,9 +843,9 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
if action.AppName == "Shuffle Tools" && (action.Name == "skip_me" || action.Name == "router" || action.Name == "route") {
err := runSkipAction(topClient, action, workflowExecution.Workflow.ID, workflowExecution.ExecutionId, workflowExecution.Authorization, "SKIPPED")
if err != nil {
log.Printf("[DEBUG] Error in skipme for %s: %s", action.Label, err)
log.Printf("[DEBUG][%s] Error in skipme for %s: %s", workflowExecution.ExecutionId, action.Label, err)
} else {
log.Printf("[INFO] Adding visited (4): %s\n", action.Label)
log.Printf("[INFO][%s] Adding visited (4): %s\n", workflowExecution.ExecutionId, action.Label)
visited = append(visited, action.ID)
executed = append(executed, action.ID)
@@ -1017,7 +1017,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
log.Printf("[INFO] %s already has status %s.", action.ID, actionResult.Status)
continue
} else {
log.Printf("[INFO] %s:%s has no status result yet. Should execute.", action.Name, action.ID)
log.Printf("[INFO][%s] %s:%s has no status result yet. Should execute.", workflowExecution.ExecutionId, action.Name, action.ID)
}
appname := action.AppName
@@ -1051,7 +1051,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
if err != nil || stats.ContainerJSONBase.State.Status != "running" {
// REMOVE
if err == nil {
log.Printf("[DEBUG] Status: %s, should kill: %s", stats.ContainerJSONBase.State.Status, identifier)
log.Printf("[DEBUG][%s] Docker Container Status: %s, should kill: %s", workflowExecution.ExecutionId, stats.ContainerJSONBase.State.Status, identifier)
err = removeContainer(identifier)
if err != nil {
log.Printf("Error killing container: %s", err)
@@ -1073,7 +1073,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
}
// marshal action and put it in there rofl
log.Printf("[INFO] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
log.Printf("[INFO][%s] Time to execute %s (%s) with app %s:%s, function %s, env %s with %d parameters.", workflowExecution.ExecutionId, action.ID, action.Label, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
actionData, err := json.Marshal(action)
if err != nil {
@@ -1096,7 +1096,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
// Sending full execution so that it won't have to load in every app
// This might be an issue if they can read environments, but that's alright
// if everything is generated during execution
log.Printf("[INFO] Deployed with CALLBACK_URL %s and BASE_URL %s", appCallbackUrl, baseUrl)
log.Printf("[INFO][%s] Deployed with CALLBACK_URL %s and BASE_URL %s", workflowExecution.ExecutionId, appCallbackUrl, baseUrl)
env := []string{
fmt.Sprintf("ACTION=%s", string(actionData)),
fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId),
@@ -1249,7 +1249,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
return
}
log.Printf("[DEBUG] Failed deploy. Downloading image %s", image)
log.Printf("[DEBUG][%s] Failed deploy. Downloading image %s", workflowExecution.ExecutionId, image)
err := downloadDockerImageBackend(topClient, image)
executed := false
if err == nil {
@@ -1328,7 +1328,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
}
}
log.Printf("[INFO] Adding visited (3): %s\n", action.Label)
log.Printf("[INFO][%s] Adding visited (3): %s\n. Actions: %d, Results: %d", workflowExecution.ExecutionId, action.Label, len(workflowExecution.Workflow.Actions), len(workflowExecution.Results))
visited = append(visited, action.ID)
executed = append(executed, action.ID)
@@ -1362,9 +1362,9 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
}
if shutdownCheck {
log.Println("[INFO] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE")
log.Println("[INFO][%s] BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE", workflowExecution.ExecutionId)
validateFinished(workflowExecution)
log.Printf("[DEBUG] Shutting down (17)")
log.Printf("[DEBUG][%s] Shutting down (17)", workflowExecution.ExecutionId)
shutdown(workflowExecution, "", "", true)
return
}
@@ -1383,9 +1383,9 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error {
results = workflowExecution.Results
startAction := workflowExecution.Start
log.Printf("[INFO] STARTACTION: %s", startAction)
log.Printf("[INFO][%s] STARTACTION: %s", workflowExecution.ExecutionId, startAction)
if len(startAction) == 0 {
log.Printf("[INFO] Didn't find execution start action. Setting it to workflow start action.")
log.Printf("[INFO][%s] Didn't find execution start action. Setting it to workflow start action.", workflowExecution.ExecutionId)
startAction = workflowExecution.Workflow.Start
}
@@ -1448,7 +1448,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error {
log.Printf("[INFO] NEXT ACTIONS: %#v\n\n", nextActions)
*/
log.Printf("[INFO] shuffle.Actions: %d + Special shuffle.Triggers: %d", len(workflowExecution.Workflow.Actions), extra)
log.Printf("[INFO][%s] shuffle.Actions: %d + Special shuffle.Triggers: %d", workflowExecution.ExecutionId, len(workflowExecution.Workflow.Actions), extra)
onpremApps := []string{}
toExecuteOnprem := []string{}
for _, action := range workflowExecution.Workflow.Actions {
@@ -2074,7 +2074,7 @@ func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExec
func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) {
if workflowExecution.ExecutionSource == "default" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" {
log.Printf("[INFO] Not sending backend info since source is default")
log.Printf("[INFO][%s] Not sending backend info since source is default", workflowExecution.ExecutionId)
return
}
@@ -2086,24 +2086,24 @@ func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) {
)
if err != nil {
log.Printf("[ERROR] Failed creating finishing request: %s", err)
log.Printf("[DEBUG] Shutting down (22)")
log.Printf("[ERROR][%s] Failed creating finishing request: %s", workflowExecution.ExecutionId, err)
log.Printf("[DEBUG][%s] Shutting down (22)", workflowExecution.ExecutionId)
shutdown(workflowExecution, "", "", false)
}
newresp, err := topClient.Do(req)
if err != nil {
log.Printf("[ERROR] Error running finishing request: %s", err)
log.Printf("[DEBUG] Shutting down (23)")
log.Printf("[ERROR][%s] Error running finishing request: %s", workflowExecution.ExecutionId, err)
log.Printf("[DEBUG][%s] Shutting down (23)", workflowExecution.ExecutionId)
shutdown(workflowExecution, "", "", false)
}
body, err := ioutil.ReadAll(newresp.Body)
//log.Printf("[INFO] BACKEND STATUS: %d", newresp.StatusCode)
if err != nil {
log.Printf("[ERROR] Failed reading body: %s", err)
log.Printf("[ERROR][%s] Failed reading body: %s", workflowExecution.ExecutionId, err)
} else {
log.Printf("[INFO] NEWRESP (from backend): %s", string(body))
log.Printf("[INFO][%s] NEWRESP (from backend): %s", workflowExecution.ExecutionId, string(body))
}
}
@@ -2209,10 +2209,71 @@ func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.Workflo
// GetLocalIP returns the non loopback local IP of the host
func getLocalIP() string {
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" {
name, err := os.Hostname()
if err != nil {
log.Printf("[ERROR] Couldn't find hostanme of worker: %s", err)
os.Exit(3)
}
log.Printf("[DEBUG] Found hostname %s since worker is running with \"run\" command", name)
return name
log.Printf("[DEBUG] Looking for IP for the external docker-network %s", swarmNetworkName)
// Different process to ensure we find the right IP.
// Necessary due to Ingress being added to docker ser
ifaces, err := net.Interfaces()
if err != nil {
log.Printf("[ERROR] FATAL: networks the container is listening in %s: %s", swarmNetworkName, err)
os.Exit(3)
}
foundIP := ""
for _, i := range ifaces {
log.Printf("NETWORK: %s", i.Name)
//If i.Name != swarmNetworkName {
// continue
//}
addrs, err := i.Addrs()
if err != nil {
log.Printf("[ERROR] FATAL: Failed getting address for listener in network %s: %s", swarmNetworkName, err)
continue
}
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
log.Printf("%s: IP: %#v", i.Name, ip)
// FIXME: Allow for IPv6 too!
//if strings.Count(ip.String(), ".") == 3 {
// foundIP = ip.String()
// break
//}
// process IP address
}
}
if len(foundIP) == 0 {
log.Printf("[ERROR] FATAL: No valid IP found for network %s. Defaulting to base IP", swarmNetworkName)
} else {
return foundIP
}
}
addrs, err := net.InterfaceAddrs()
if err != nil {
return ""
}
for _, address := range addrs {
// check the address type and if it is not a loopback the display it
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
@@ -2221,6 +2282,7 @@ func getLocalIP() string {
}
}
}
return ""
}
@@ -2619,7 +2681,7 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action
parsedRequest.Url = fmt.Sprintf("%s:%d", parsedBaseurl, baseport)
}
log.Printf("[DEBUG] Should add a baseurl for the app to get back to: %s", parsedRequest.Url)
log.Printf("[DEBUG][%s] Should add a baseurl for the app to get back to: %s", workflowExecution.ExecutionId, parsedRequest.Url)
}
// FIXME: Swapping because this was confusing during dev
@@ -2632,7 +2694,7 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action
if len(hostname) > 0 {
parsedRequest.BaseUrl = fmt.Sprintf("http://%s:%d", hostname, baseport)
//parsedRequest.BaseUrl = fmt.Sprintf("http://shuffle-workers:%d", baseport)
log.Printf("[DEBUG] Changing hostname to local hostname in Docker network for WORKER URL: %s", parsedRequest.BaseUrl)
log.Printf("[DEBUG][%s] Changing hostname to local hostname in Docker network for WORKER URL: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl)
}
data, err := json.Marshal(parsedRequest)
@@ -2643,7 +2705,7 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action
//streamUrl := fmt.Sprintf("%s:%d/api/v1/run", parsedBaseurl, port)
streamUrl := fmt.Sprintf("http://%s:%d/api/v1/run", appName, port)
log.Printf("[DEBUG] Worker URL: %s, Backend URL: %s, Target App: %s", parsedRequest.BaseUrl, parsedRequest.Url, streamUrl)
log.Printf("[DEBUG][%s] Worker URL: %s, Backend URL: %s, Target App: %s", workflowExecution.ExecutionId, parsedRequest.BaseUrl, parsedRequest.Url, streamUrl)
req, err := http.NewRequest(
"POST",
streamUrl,
@@ -2667,7 +2729,7 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action
log.Printf("[ERROR] Failed reading body: %s", err)
return err
} else {
log.Printf("[INFO] NEWRESP (from app): %s", string(body))
log.Printf("[INFO][%s] NEWRESP (from app): %s", workflowExecution.ExecutionId, string(body))
}
// FIXME: Remove