Merge pull request #1596 from Shuffle/nightly

Bugfixes for 2.0.0
This commit is contained in:
Frikky
2025-01-13 14:27:50 +01:00
committed by GitHub
28 changed files with 2427 additions and 1787 deletions
+5 -8
View File
@@ -4,7 +4,7 @@ on:
workflow_dispatch:
push:
branches:
- main
- nightly
paths:
- "**"
- "!.github/**"
@@ -20,19 +20,19 @@ jobs:
include:
- app: frontend
path: frontend
version: 2.0.0
version: nightly
experimental: true
- app: backend
path: backend
version: 2.0.0
version: nightly
experimental: true
- app: orborus
path: functions/onprem/orborus
version: 2.0.0
version: nightly
experimental: true
- app: worker
path: functions/onprem/worker
version: 2.0.0
version: nightly
experimental: true
steps:
- name: Checkout
@@ -74,11 +74,8 @@ jobs:
cache-to: type=local,dest=/tmp/.buildx-cache
tags: |
ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }}
ghcr.io/shuffle/shuffle-${{ matrix.app }}:latest
${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }}
${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:latest
frikky/shuffle-${{ matrix.app }}:${{ matrix.version }}
frikky/shuffle-${{ matrix.app }}:latest
frikky/shuffle:${{ matrix.app }}
- name: Image digest
+1 -1
View File
@@ -3,7 +3,7 @@ on:
release:
types: [published]
branches:
- 2.0.0
- nightly
jobs:
main:
+1
View File
@@ -6,6 +6,7 @@ Shuffle Automation
[![CodeQL](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/codeql-analysis.yml)
[![Autobuild](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml/badge.svg?branch=launch)](https://github.com/Shuffle/Shuffle/actions/workflows/dockerbuild.yaml)
[![Deploy to AWS](https://d1.awsstatic.com/cloudformation-deploy-to-aws-button.png)](https://console.aws.amazon.com/cloudformation/home?#/stacks/new?stackName=Shuffle-Instance&templateURL=https://shuffle-public-amis.s3.eu-north-1.amazonaws.com/template.yaml)
</h1><h4 align="center">
+9 -9
View File
@@ -20,8 +20,8 @@ require (
github.com/gorilla/mux v1.8.1
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.6.90
golang.org/x/crypto v0.31.0
github.com/shuffle/shuffle-shared v0.6.94
golang.org/x/crypto v0.22.0
google.golang.org/api v0.176.1
google.golang.org/grpc v1.63.2
gopkg.in/src-d/go-git.v4 v4.13.1
@@ -116,15 +116,15 @@ require (
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/oauth2 v0.19.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/tools v0.18.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
+18 -16
View File
@@ -334,6 +334,8 @@ github.com/sendgrid/sendgrid-go v3.14.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdR
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/shuffle-shared v0.6.77 h1:KKtM50xW2DLuRHINxhp3uXrNH0AhiwkeiiU93a8fB3A=
github.com/shuffle/shuffle-shared v0.6.77/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/shuffle/shuffle-shared v0.6.90 h1:FzIYtEt44eWgEsW/9tj2ki7qq8FEm/HWXUok+THp72M=
github.com/shuffle/shuffle-shared v0.6.90/go.mod h1:RAJiSFjmuKmijKTbbEf9A6Ojb+3/te7g71lED7JjPus=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@@ -408,8 +410,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -438,8 +440,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -466,8 +468,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -484,8 +486,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -515,16 +517,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -536,8 +538,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
@@ -572,8 +574,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+4 -4
View File
@@ -1,6 +1,6 @@
services:
frontend:
image: ghcr.io/shuffle/shuffle-frontend:1.4.0
image: ghcr.io/shuffle/shuffle-frontend:1.4.2
container_name: shuffle-frontend
hostname: shuffle-frontend
ports:
@@ -14,7 +14,7 @@ services:
depends_on:
- backend
backend:
image: ghcr.io/shuffle/shuffle-backend:1.4.0
image: ghcr.io/shuffle/shuffle-backend:1.4.2
container_name: shuffle-backend
hostname: ${BACKEND_HOSTNAME}
# Here for debugging:
@@ -53,9 +53,9 @@ services:
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
- SHUFFLE_STATS_DISABLED=true
# - SHUFFLE_SWARM_CONFIG=run
- SHUFFLE_SWARM_CONFIG=run
- SHUFFLE_LOGS_DISABLED=true
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:1.4.0
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:1.4.2
env_file: .env
restart: unless-stopped
security_opt:
+128 -94
View File
@@ -11,6 +11,8 @@ import {
Button,
Stack,
Avatar,
Skeleton,
Tooltip,
} from '@mui/material';
import CloseIcon from '@mui/icons-material/Close';
@@ -22,8 +24,8 @@ import OpenInNewIcon from '@mui/icons-material/OpenInNew';
import LaunchIcon from '@mui/icons-material/Launch';
import CheckCircleIcon from '@mui/icons-material/CheckCircle';
import { CloudDownloadOutlined } from '@mui/icons-material';
import { findSpecificApp } from './AppFramework';
import theme from "../theme";
import { findSpecificApp } from '../components/AppFramework.jsx';
import theme from "../theme.jsx";
import YAML from 'yaml';
import { toast } from 'react-toastify';
import { Link } from 'react-router-dom';
@@ -38,6 +40,7 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
const [inputUsecase, setInputUsecase] = useState({})
const [latestUsecase, setLatestUsecase] = useState([])
const [foundAppUsecase, setFoundAppUsecase] = useState({})
const [usecaseLoading, setUsecaseLoading] = useState(false)
const navigate = useNavigate();
const parseUsecase = (subcase) => {
const srcdata = findSpecificApp(frameworkData, subcase.type)
@@ -142,6 +145,19 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
})
setLatestUsecase(newUsecases)
if (newUsecases?.length > 0) {
const foundCategory = newUsecases?.find((category) =>
category?.list?.some((subcase) => subcase?.srcapp === app?.name || subcase?.dstapp === app?.name)
);
const foundSubcase = foundCategory?.list?.find(
(subcase) => subcase?.srcapp === app?.name || subcase?.dstapp === app?.name
);
setFoundAppUsecase(foundSubcase);
}
setUsecaseLoading(false)
// Matching workflows with usecases
if (responseJson.success !== false) {
if (workflows !== undefined && workflows !== null && workflows.length > 0) {
@@ -275,23 +291,12 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
useEffect(() => {
setUsecaseLoading(true)
getAvailableWorkflows()
getFramework()
}, [app])
useEffect(() => {
const foundCategory = latestUsecase?.find((category) =>
category?.list?.some((subcase) => subcase?.srcapp === app?.name || subcase?.dstapp === app?.name)
);
const foundSubcase = foundCategory?.list?.find(
(subcase) => subcase?.srcapp === app?.name || subcase?.dstapp === app?.name
);
setFoundAppUsecase(foundSubcase);
}, [latestUsecase])
const downloadApp = (inputdata) => {
const id = inputdata.id;
@@ -389,9 +394,8 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
newAppname = newAppname?.replaceAll("_", " ");
}
var canEditApp = userdata.admin === "true" || userdata?.id === app?.owner || app?.owner === "" || (userdata.admin === "true" && userdata.active_org.id === app?.reference_org) || !app?.generated
var canEditApp = userdata !== undefined && (userdata?.admin === "true" || userdata?.id === app?.owner || app?.owner === "" || (userdata?.admin === "true" && userdata?.active_org?.id === app?.reference_org)) || !app?.generated
return (
<Dialog
@@ -501,28 +505,37 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
app?.private_id !== undefined &&
app?.private_id?.length > 0 &&
app?.generated ? (
<Button
variant="contained"
<Tooltip title="Download OpenAPI"
placement="top"
arrow
sx={{
bgcolor: '#494949',
'&:hover': { bgcolor: '#494949' },
textTransform: 'none',
borderRadius: 1,
minWidth: '45px',
width: '45px',
height: '40px',
padding: 2,
color: "#fff",
fontFamily: theme?.typography?.fontFamily
}}
onClick={(event) => {
event.preventDefault();
event.stopPropagation();
downloadApp(app);
}}
>
<CloudDownloadOutlined />
</Button>) : null}
<Button
variant="contained"
sx={{
bgcolor: '#494949',
'&:hover': { bgcolor: '#494949' },
textTransform: 'none',
borderRadius: 1,
minWidth: '45px',
width: '45px',
height: '40px',
padding: 2,
color: "#fff",
fontFamily: theme?.typography?.fontFamily
}}
onClick={(event) => {
event.preventDefault();
event.stopPropagation();
downloadApp(app);
}}
>
<CloudDownloadOutlined />
</Button>
</Tooltip>
) : null}
<Button
variant="contained"
sx={{
@@ -655,21 +668,32 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
justifyContent: "start",
width: "100%"
}}>
<div style={{
fontFamily: theme?.typography?.fontFamily,
fontSize: "16px",
color: "#fff",
marginBottom: "16px",
fontWeight: 600
}}>
{
(foundAppUsecase?.srcapp !== undefined && foundAppUsecase?.dstapp !== undefined) ? (
"Connect " + foundAppUsecase?.srcapp?.replaceAll("_", " ") + " to " + foundAppUsecase?.dstapp?.replaceAll("_", " ")
) : (
"Connect " + app?.name + " to any tool"
)
}
</div>
{usecaseLoading ? (
<Skeleton
variant="text"
width="55%"
sx={{
fontSize: "16px",
mb: "16px",
}}
/>
) : (
<div style={{
fontFamily: theme?.typography?.fontFamily,
fontSize: "16px",
color: "#fff",
marginBottom: "16px",
fontWeight: 600
}}>
{
(foundAppUsecase?.srcapp !== undefined && foundAppUsecase?.dstapp !== undefined) ? (
"Connect " + foundAppUsecase?.srcapp?.replaceAll("_", " ") + " to " + foundAppUsecase?.dstapp?.replaceAll("_", " ")
) : (
"Connect " + app?.name.replaceAll("_", " ") + " to any tool"
)
}
</div>
)}
<Box sx={{
bgcolor: '#2F2F2F',
@@ -679,49 +703,56 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
alignItems: 'center',
mb: 3
}}>
<Stack direction="row" spacing={-1}>
{
foundAppUsecase === undefined ? (
<Avatar sx={{ width: 32, height: 32, bgcolor: 'background.paper', border: 1, borderColor: 'divider' }}>
<Search sx={{ color: 'text.primary', zIndex: 10, fontSize: 18 }} />
</Avatar>
) : (
<Avatar
src={foundAppUsecase?.srcimg}
sx={{
width: 32,
height: 32,
bgcolor: 'background.paper',
border: 1,
borderColor: 'divider',
zIndex: 10
}}
/>
)
}
{
foundAppUsecase === undefined ? (
<Avatar sx={{ width: 32, height: 32, bgcolor: 'background.paper', border: 1, borderColor: 'divider' }}>
<AddIcon sx={{ color: 'text.primary', zIndex: 10, fontSize: 18 }} />
</Avatar>
) : (
<Avatar
src={foundAppUsecase?.dstimg}
sx={{
width: 32,
height: 32,
bgcolor: 'background.paper',
border: 1,
borderColor: 'divider',
zIndex: 10
}}
/>
)
}
</Stack>
<Typography sx={{ ml: 2, fontSize: "16px", letterSpacing: "0.5px" }}>
{foundAppUsecase?.name || "Search for a Usecase"}
</Typography>
{usecaseLoading ? (
<Stack direction="row" spacing={2} alignItems="center" sx={{ width: '100%' }}>
<Stack direction="row" spacing={-1}>
<Skeleton variant="circular" width={32} height={32} />
<Skeleton variant="circular" width={32} height={32} />
</Stack>
<Skeleton variant="text" sx={{ flexGrow: 1 }} width={200} />
</Stack>
) : (
<>
<Stack direction="row" spacing={-1}>
{foundAppUsecase?.srcapp === undefined ? (
<Avatar sx={{ width: 32, height: 32, bgcolor: 'background.paper', border: 1, borderColor: 'divider' }} src={app?.image_url || app?.large_image}>
</Avatar>
) : (
<Avatar
src={foundAppUsecase?.srcimg}
sx={{
width: 32,
height: 32,
bgcolor: 'background.paper',
border: 1,
borderColor: 'divider',
zIndex: 10
}}
/>
)}
{foundAppUsecase?.dstapp === undefined ? (
<Avatar sx={{ width: 32, height: 32, bgcolor: 'background.paper', border: 1, borderColor: 'divider' }}>
<AddIcon sx={{ color: 'text.primary', zIndex: 10, fontSize: 18 }} />
</Avatar>
) : (
<Avatar
src={foundAppUsecase?.dstimg}
sx={{
width: 32,
height: 32,
bgcolor: 'background.paper',
border: 1,
borderColor: 'divider',
zIndex: 10
}}
/>
)}
</Stack>
<Typography sx={{ ml: 2, fontSize: "16px", letterSpacing: "0.5px" }}>
{foundAppUsecase?.name || "Search for a Usecase"}
</Typography>
</>
)}
</Box>
</div>
@@ -742,10 +773,13 @@ const AppModal = ({ open, onClose, app, globalUrl }) => {
minWidth: '200px'
}}
onClick={() => {
navigate("/usecases2")
navigate("/usecases")
}}
disabled={usecaseLoading}
>
Find a Usecase
{
(foundAppUsecase !== undefined && foundAppUsecase !== null && usecaseLoading === false) ? "See usecase" : "Find a Usecase"
}
</Button>
</div>
</DialogContent>
+9 -9
View File
@@ -536,15 +536,16 @@ const AppSelection = props => {
})}
</Grid>
</div>
{
!isAppPage && (
<>
{!moreButton ? (
<div style={{ width: "100%", marginLeft: isMobile ? 80 : 200, marginBottom: 20, textAlign: isMobile ? "center" : null }}>
<Link style={{ color: "#FF8444" }} onClick={() => {
setMoreButton(true)
setTimeout(() => {
navigate("/welcome?tab=2")
if (isAppPage) {
navigate("/apps?tab=all_apps")
} else {
navigate("/welcome?tab=2")
}
}, 250)
}}
>See More Apps</Link>
@@ -552,15 +553,14 @@ const AppSelection = props => {
<div style={{ flexDirection: "row", width: isMobile ? 340 : null, textAlign: isMobile ? "center" : null }}>
<Button variant="contained" type="submit" fullWidth style={bottomButtonStyle} onClick={() => {
navigate("/usecases2")
setActiveStep(2)
navigate("/usecases")
if(!isAppPage) {
setActiveStep(2)
}
}}>
See usecases
</Button>
</div>
</>
)
}
</div>
</Fade>
)
+1
View File
@@ -537,6 +537,7 @@ const CacheView = memo((props) => {
padding: "15px 5px",
maxHeight: 300,
verticalAlign: "middle",
maxWidth: 300,
}}
primary={validate.valid ?
<ReactJson
@@ -133,14 +133,18 @@ const DetectionExplorer = (props) => {
const handleConnectClick = () => {
if (detectionWorkflowId !== "") {
console.log("Already have a workflow ID for this detection")
//toast.info(`Already have a detection workflow for ${detectionInfo?.category}`)
// FIXME: Show the Usecase UI for how to fix the workflow(s)
// Instead loading full workflow and showing it directly? Hmm
//toast.warn("Please reload the UI to load the detection status")
return
//return
}
if (isDetectionActive) {
return
console.log("Already connected")
//toast.info(`Connected to ${detectionInfo?.category}`)
//return
}
if (detectionInfo.category === undefined || detectionInfo.category === null) {
@@ -276,7 +280,7 @@ const DetectionExplorer = (props) => {
}}
>
<Typography variant="h6" component="div">
{detectionInfo?.title} {filteredRules === undefined || filteredRules === null ? null : `(${filteredRules?.length} rules)`}
{detectionInfo?.title} {filteredRules === undefined || filteredRules === null ? null : `(${filteredRules?.length} rule${filteredRules?.length > 1 ? "s" : ""})`}
</Typography>
<div style={{display: "flex", }}>
+424 -32
View File
@@ -22,6 +22,9 @@ import {
DialogActions,
Typography,
Skeleton,
Checkbox,
Chip,
Menu,
} from "@mui/material";
import {
@@ -35,6 +38,7 @@ import {
Publish as PublishIcon,
Clear as ClearIcon,
Add as AddIcon,
SelectAll,
} from "@mui/icons-material";
import Dropzone from "../components/Dropzone.jsx";
@@ -61,6 +65,14 @@ const Files = memo((props) => {
const [downloadBranch, setDownloadBranch] = React.useState("main");
const [downloadFolder, setDownloadFolder] = React.useState("translation_standards");
const [contentLoading, setContentLoading] = React.useState(false)
const [selectAllChecked, setSelectAllChecked] = React.useState(false)
const [selectedFiles, setSelectedFiles] = useState([]);
const [selectedFileId, setSelectedFileId] = useState([])
const [showFileCategoryPopup, setShowFileCategoryPopup] = useState(false)
const [updateToThisCategory, setUpdateToThisCategory] = useState("")
const [showDistributionPopup, setShowDistributionPopup] = useState(false)
const [selectedSubOrg, setSelectedSubOrg] = useState([])
const [fileIdSelectedForDistribution, setFileIdSelectedForDistribution] = useState("")
//const alert = useAlert();
const allowedFileTypes = ["txt", "py", "yaml", "yml","json", "html", "js", "csv", "log", "eml", "msg", "md", "xml", "sh", "bat", "ps1", "psm1", "psd1", "ps1xml", "pssc", "psc1", "response"]
var upload = "";
@@ -80,6 +92,56 @@ const Files = memo((props) => {
}
const changeDistribution = (id, selectedSubOrg) => {
editFileConfig(id, "suborg_distribute", [...new Set(selectedSubOrg)])
}
const editFileConfig = (id, parentAction, selectedSubOrg) => {
const data = {
id: id,
action: parentAction !== undefined && parentAction !== null ? parentAction : "change_category",
selected_suborgs: selectedSubOrg,
}
const url = globalUrl + "/api/v1/files/" + id + "/config";
fetch(url, {
mode: "cors",
method: "POST",
body: JSON.stringify(data),
credentials: "include",
crossDomain: true,
withCredentials: true,
headers: {
"Content-Type": "application/json; charset=utf-8",
},
})
.then((response) =>
response.json().then((responseJson) => {
if (responseJson["success"] === false) {
toast("Failed overwriting files");
} else {
toast("Successfully updated file!");
setTimeout(() => {
getFiles();
}, 1000);
}
})
)
.catch((error) => {
toast("Err: " + error.toString());
});
};
const handleFileCheckboxChange = (index) => {
setSelectedFiles((prevSelected) => {
const updatedSelected = [...prevSelected];
updatedSelected[index] = !updatedSelected[index];
return updatedSelected;
});
};
const runUpdateText = (text) =>{
fetch(`${globalUrl}/api/v1/files/${openFileId}/edit`, {
method: "PUT",
@@ -137,6 +199,7 @@ const Files = memo((props) => {
if (responseJson.files !== undefined && responseJson.files !== null) {
setFiles(responseJson.files);
setShowLoader(false)
setShowDistributionPopup(false)
} else if (responseJson.list !== undefined && responseJson.list !== null) {
// Set the "namespace" field in all items
if (namespace !== undefined && namespace !== null) {
@@ -152,6 +215,7 @@ const Files = memo((props) => {
} else {
setFiles([]);
setShowLoader(false)
setShowDistributionPopup(false)
}
if (namespace === undefined || namespace === null || namespace === "default") {
@@ -256,7 +320,7 @@ const Files = memo((props) => {
zIndex: 1000,
minWidth: "800px",
minHeight: "320px",
overflow: "hidden",
overflow: "auto",
'& .MuiDialogContent-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
@@ -400,6 +464,127 @@ const Files = memo((props) => {
</Dialog>
: null
const handleSelectSubOrg = (id, action) => {
if (action === "all") {
const childOrgs = userdata.orgs.filter(
(data) => data.creator_org === userdata.active_org.id
);
setSelectedSubOrg((prev) => {
if (prev.length === childOrgs.length) {
// If all child orgs are already selected, clear the selection
return [];
} else {
// Otherwise, select all child org IDs
return childOrgs.map((data) => data.id);
}
});
} else if (action === "none") {
setSelectedSubOrg([]);
} else {
setSelectedSubOrg((prev) => {
if (prev.includes(id)) {
return prev.filter((data) => data !== id);
} else {
return [...prev, id];
}
});
}
};
const fileDistributionModal = showDistributionPopup ? (
<Dialog
open={showDistributionPopup}
onClose={() => setShowDistributionPopup(false)}
PaperProps={{
sx: {
borderRadius: theme?.palette?.DialogStyle?.borderRadius,
border: theme?.palette?.DialogStyle?.border,
fontFamily: theme?.typography?.fontFamily,
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
zIndex: 1000,
minWidth: "600px",
minHeight: "320px",
overflow: "auto",
'& .MuiDialogContent-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
'& .MuiDialogTitle-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
'& .MuiDialogActions-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
},
}}
>
<DialogTitle>
<div style={{ color: "rgba(255,255,255,0.9)" }}>
Select sub-org to distribute files
</div>
</DialogTitle>
<DialogContent style={{ color: "rgba(255,255,255,0.65)" }}>
<MenuItem value="none" onClick={()=> {handleSelectSubOrg(null, "none")}}>None</MenuItem>
<MenuItem value="all" onClick={()=> {handleSelectSubOrg(null, "all")}}>All</MenuItem>
{userdata.orgs.map((data, index) => {
if (data.creator_org !== userdata.active_org.id) {
return null;
}
const imagesize = 22;
const imageStyle = {
width: imagesize,
height: imagesize,
pointerEvents: "none",
marginRight: 10,
marginLeft: data.id === userdata.active_org.id ? 0 : 20,
};
const image = data.image === "" ? (
<img alt={data.name} src={theme.palette.defaultImage} style={imageStyle} />
) : (
<img alt={data.name} src={data.image} style={imageStyle} />
);
return (
<MenuItem
key={index}
value={data.id}
onClick={() => handleSelectSubOrg(data.id)}
style={{ display: "flex", alignItems: "center" }}
>
<Checkbox
checked={selectedSubOrg.includes(data.id)}
/>
{image}
<span style={{ marginLeft: 8 }}>{data.name}</span>
</MenuItem>
);
})}
<div style={{ display: "flex", marginTop: 20 }}>
<Button
style={{ borderRadius: "2px", textTransform: 'none', fontSize:16, color: "#ff8544" }}
onClick={() => setShowDistributionPopup(false)}
color="primary"
>
Cancel
</Button>
<Button
variant="contained"
style={{ borderRadius: "2px", textTransform: 'none', fontSize:16, color: "#1a1a1a", backgroundColor: "#ff8544", marginLeft: 10 }}
onClick={() => {
changeDistribution(fileIdSelectedForDistribution, selectedSubOrg);
}}
color="primary"
>
Submit
</Button>
</div>
</DialogContent>
</Dialog>
): null
const deleteFile = (file) => {
fetch(globalUrl + "/api/v1/files/" + file.id, {
method: "DELETE",
@@ -636,7 +821,7 @@ const Files = memo((props) => {
setTimeout(() => {
getFiles()
}, 2500);
}, 3000);
};
const uploadFile = (e) => {
@@ -649,6 +834,68 @@ const Files = memo((props) => {
uploadFiles(files);
};
const handleUpdateFileCategory = (namespace) => {
if (selectedFiles.length === 0 && !selectAllChecked) {
toast("Please select files to update category")
return
}
if (namespace === undefined || namespace === null || namespace === "") {
toast("Please select a category to update files to")
return
}
const url = globalUrl + `/api/v1/files/namespaces/${namespace}/share`
const data = {
SelectedFiles: selectedFileId,
}
setShowLoader(true)
fetch(url, {
mode: "cors",
method: "POST",
body: JSON.stringify(data),
credentials: "include",
crossDomain: true,
withCredentials: true,
headers: {
"Content-Type": "application/json; charset=utf-8",
},
})
.then((response) =>
response.json().then((responseJson) => {
if (responseJson["success"] === false) {
toast("Failed overwriting files");
} else {
setSelectAllChecked(false)
setSelectedFiles([])
setSelectedFileId([])
setShowFileCategoryPopup(false)
setSelectedCategory(namespace)
setTimeout(() => {
getFiles();
toast("Successfully updated file!");
if (window.location.search.includes("category=")) {
const newurl = window.location.href.replace(/category=[^&]+/, `category=${namespace}`)
window.history.pushState({ path: newurl }, "", newurl)
} else {
window.history.pushState({ path: window.location.href }, "", `${window.location.href}&category=${namespace}`)
}
}, 1000);
}
}
))
.catch((error) => {
toast("Err: " + error.toString());
});
}
return (
<Dropzone
style={{
@@ -659,6 +906,7 @@ const Files = memo((props) => {
}}
onDrop={uploadFile}
>
{fileDistributionModal}
<div style={{width: "100%", minHeight: 1100, boxSizing: 'border-box', padding: "27px 10px 19px 27px", height:"100%", backgroundColor: '#212121',borderTopRightRadius: '8px', borderBottomRightRadius: 8, borderLeft: "1px solid #494949", }}>
<div style={{height: "100%", maxHeight: 1700,overflowY: 'auto', scrollbarColor: '#494949 transparent', scrollbarWidth: 'thin'}}>
@@ -739,13 +987,18 @@ const Files = memo((props) => {
}}
value={selectedCategory}
onChange={(event) => {
if (selectAllChecked || selectedFiles.length > 0) {
setUpdateToThisCategory(event.target.value)
setShowFileCategoryPopup(true)
return
}
setSelectedCategory(event.target.value)
if (event.target.value === "all" || event.target.value === "default") {
getFiles()
} else {
getFiles(event.target.value)
}
// Add it to the url as a query
if (window.location.search.includes("category=")) {
@@ -768,14 +1021,41 @@ const Files = memo((props) => {
);
})}
</Select>
<Dialog
PaperProps={{
sx: {
borderRadius: theme?.palette?.DialogStyle?.borderRadius,
border: theme?.palette?.DialogStyle?.border,
fontFamily: theme?.typography?.fontFamily,
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
zIndex: 1000,
'& .MuiDialogContent-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
'& .MuiDialogTitle-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
'& .MuiDialogActions-root': {
backgroundColor: theme?.palette?.DialogStyle?.backgroundColor,
},
},
}} open={showFileCategoryPopup} onClose={() => setShowFileCategoryPopup(false)}>
<DialogTitle>File Categories</DialogTitle>
<DialogContent>
Please note that your selected files ({selectedFileId?.length}) will be moved to the <kbd>{updateToThisCategory}</kbd> category.
</DialogContent>
<DialogActions>
<Button onClick={() => setShowFileCategoryPopup(false)} style={{fontSize: 16, textTransform: 'none'}}>Close</Button>
<Button onClick={() => handleUpdateFileCategory(updateToThisCategory)} style={{fontSize: 16, textTransform: 'none', color: "#1a1a1a", backgroundColor: "#ff8544"}}>Update</Button>
</DialogActions>
</Dialog>
</FormControl>
) : null}
<div style={{display: "inline-flex", position:"relative", top: 8}}>
{renderTextBox ?
<Tooltip title={"Close"} style={{}} aria-label={""}>
<Button
style={{ marginLeft: 5, marginRight: 15 }}
style={{ marginLeft: 5, marginRight: 15, height: 35, borderRadius: 4, backgroundColor: "#494949", textTransform: 'none', fontSize: 16, color: "#f1f1f1" }}
color="primary"
onClick={() => {
setRenderTextBox(false);
@@ -803,10 +1083,24 @@ const Files = memo((props) => {
{renderTextBox && <TextField
onKeyPress={(event)=>{
handleKeyDown(event);
if(event.key === 'Enter' && selectedFileId.length > 0){
setShowFileCategoryPopup(true)
setUpdateToThisCategory(event.target.value)
}
}}
style={{
height: 35,
width: 200,
marginTop: 0,
}}
InputProps={{
style: {
color: "white",
height: 35,
fontSize: 16,
borderRadius: 4,
paddingTop: 0,
},
}}
color="primary"
@@ -816,7 +1110,6 @@ const Files = memo((props) => {
defaultValue={""}
autoFocus
/>}</div>
<ShuffleCodeEditor
isCloud={isCloud}
expansionModalOpen={openEditor}
@@ -852,30 +1145,70 @@ const Files = memo((props) => {
tableLayout: "auto",
display: "table",
minWidth: 800,
overflowX: "auto"
overflowX: "auto",
paddingBottom: 0,
}}
>
<ListItem style={{width:isSelectedFiles?"100%":null, borderBottom:isSelectedFiles?"1px solid #494949":null, display: 'table-row'}}>
{["Name", "Workflow", "Md5", "Status", "Filesize", "Actions"].map((header, index) => (
<ListItemText
key={index}
primary={header}
style={{
display: "table-cell",
padding: index === 0 ? "0px 8px 8px 15px": "0px 8px 8px 8px",
whiteSpace: "nowrap",
textOverflow: "ellipsis",
borderBottom: "1px solid #494949",
verticalAlign: "middle"
}}
primaryTypographyProps={{
<ListItem
style={{
borderBottom: "1px solid #494949" ,
display: "table-row"
}}
>
{[
<Tooltip title={"Select all files"} style={{}} aria-label={""}>
<Checkbox
sx={{padding: 0}}
onChange={() => {
setSelectAllChecked((prev) => !prev);
setSelectedFiles((prev) => {
if (prev.length === files.length) {
return []
} else {
return files.map((_, index) => !prev.includes(index))
}
})
if (selectAllChecked) {
setSelectedFileId([])
} else {
setSelectedFileId(
files
.filter((file) => file.namespace === selectedCategory)
.map((file) => file.id)
);
}
}}
/>
</Tooltip>,
"Name",
"Workflow",
"Md5",
"Status",
"Filesize",
"Actions",
"Distribution"
]
.filter(Boolean)
.map((header, index) => (
<ListItemText
key={index}
primary={header}
style={{
display: "table-cell",
padding: index === 0 ? "0px 8px 8px 15px" : "0px 8px 8px 8px",
whiteSpace: "nowrap",
textOverflow: "ellipsis",
borderBottom: "1px solid #494949",
verticalAlign: "middle"
}}
primaryTypographyProps={{
style: {
paddingLeft: 10,
paddingLeft: 10
}
}}
/>
))}
</ListItem>
}}
/>
))}
</ListItem>
{showLoader ?
[...Array(6)].map((_, rowIndex) => (
<ListItem
@@ -885,7 +1218,7 @@ const Files = memo((props) => {
backgroundColor: "#212121",
}}
>
{Array(6)
{Array(8)
.fill()
.map((_, colIndex) => (
<ListItemText
@@ -928,7 +1261,7 @@ const Files = memo((props) => {
if (index % 2 === 0) {
bgColor = isSelectedFiles ? "#1A1A1A":"#1f2023";
}
const isDistributed = file?.suborg_distribution?.length > 0 ? true : false;
const filenamesplit = file.filename.split(".")
const iseditable = file.filesize < 2000000 && file.status === "active" && allowedFileTypes.includes(filenamesplit[filenamesplit.length-1])
return (
@@ -953,6 +1286,26 @@ const Files = memo((props) => {
primary={new Date(file.updated_at * 1000).toISOString()}
/>
*/}
<ListItemText
style={{
display: 'table-cell',
overflow: "hidden",
textAlign: "center",
}}
>
<Checkbox
style={{ padding: 0 }}
disabled={file.org_id !== selectedOrganization.id}
checked={!!selectedFiles[index] || selectAllChecked}
onChange={() => {handleFileCheckboxChange(index); setSelectedFileId(prev => {
if (prev.includes(file.id)) {
return prev.filter((item) => item !== file.id)
} else {
return [...prev, file.id]
}
})}}
/>
</ListItemText>
<ListItemText
primaryTypographyProps={{
style: {
@@ -1064,7 +1417,7 @@ const Files = memo((props) => {
>
<span>
<IconButton
disabled={!iseditable}
disabled={!iseditable || file.org_id !== selectedOrganization.id}
style = {{padding: "6px", }}
onClick={() => {
setOpenEditor(true)
@@ -1137,7 +1490,6 @@ const Files = memo((props) => {
<IconButton
style = {{padding: "6px"}}
onClick={() => {
console.log("file is : ", file)
navigator.clipboard.writeText(file.id);
document.execCommand("copy");
@@ -1154,7 +1506,7 @@ const Files = memo((props) => {
>
<span>
<IconButton
disabled={file.status !== "active"}
disabled={file.status !== "active" || file.org_id !== selectedOrganization.id}
style={{ padding: "6px" }}
onClick={() => {
deleteFile(file);
@@ -1166,7 +1518,7 @@ const Files = memo((props) => {
viewBox="0 0 24 24"
xmlns="http://www.w3.org/2000/svg"
style={{
stroke: file.status === "active" ? "#fd4c62" : "#c8c8c8",
stroke: file.status === "active" && file.org_id === selectedOrganization.id ? "#fd4c62" : "#c8c8c8",
}}
>
<path
@@ -1192,6 +1544,46 @@ const Files = memo((props) => {
// overflow: "hidden",
}}
/>
<ListItemText primaryTypographyProps={{
style: {
padding: 8
}
}} style={{ display: "table-cell", textAlign: 'center', verticalAlign: 'middle'}} >
{selectedOrganization.id !== undefined && file?.org_id !== selectedOrganization.id ?
<Tooltip
title="Parent organization controlled file. You can use, but not modify this file. Contact an admin of your parent organization if you need changes to this."
placement="top"
>
<Chip
label={"Parent"}
variant="contained"
color="secondary"
style={{display: "table-cell",}}
/>
</Tooltip>
:
<Tooltip
title="Distributed to sub-organizations. This means the sub organizations can use this file, but can not modify it."
placement="top"
>
<Checkbox
disabled={selectedOrganization.creator_org !== undefined && selectedOrganization.creator_org !== null && selectedOrganization.creator_org !== "" ? true : false}
checked={isDistributed}
style={{ }}
color="secondary"
onClick={() => {
setShowDistributionPopup(true)
if(file?.suborg_distribution?.length > 0){
setSelectedSubOrg(file.suborg_distribution)
}else{
setSelectedSubOrg([])
}
setFileIdSelectedForDistribution(file.id)
}}
/>
</Tooltip>
}
</ListItemText>
</ListItem>
);
})
+50 -32
View File
@@ -58,6 +58,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => {
const navigate = useNavigate();
const {setLeftSideBarOpenByClick, leftSideBarOpenByClick, setSearchBarModalOpen, searchBarModalOpen} = useContext(Context);
const [expandLeftNav, setExpandLeftNav] = useState(false);
const [activeOrgName, setActiveOrgName] = useState(
userdata?.active_org?.name || "Select Organziation"
@@ -261,17 +262,20 @@ useEffect(() => {
},[currentPath]);
useEffect(() => {
UpdateTabStatus();
const expandLeftNav1 = localStorage.getItem("expandLeftNav");
UpdateTabStatus()
const expandLeftNav1 = localStorage.getItem("expandLeftNav")
if (expandLeftNav1 === "false") {
setLeftSideBarOpenByClick(false);
setLeftSideBarOpenByClick(false);
setLeftSideBarOpenByClick(false)
} else {
setLeftSideBarOpenByClick(true);
setLeftSideBarOpenByClick(true);
setExpandLeftNav(true);
const currentLocation = window?.location?.pathname
if (currentLocation?.includes('/workflows/')) {
} else {
setLeftSideBarOpenByClick(true)
setExpandLeftNav(true)
}
}
}, []);
}, [])
const getAvailableWorkflows = useCallback((amount) => {
@@ -533,7 +537,7 @@ useEffect(() => {
<Divider style={{ marginBottom: 10, }} />
<Typography color="textSecondary" align="center" style={{ marginTop: 5, marginBottom: 5, fontSize: 18 }}>
Version: 2.0.0-beta
Version: 2.0.0-rc2
</Typography>
</Menu>
</span>
@@ -591,6 +595,10 @@ useEffect(() => {
org_id: orgId,
};
if (userdata?.active_org?.id === orgId) {
return
}
localStorage.setItem("globalUrl", "");
localStorage.setItem("getting_started_sidebar", "open");
@@ -662,7 +670,8 @@ useEffect(() => {
};
const getRegionTag = (region_url) => {
let regiontag = "UK";
//let regiontag = "UK";
let regiontag = "EU";
if (
region_url !== undefined &&
region_url !== null &&
@@ -676,7 +685,7 @@ useEffect(() => {
if (regiontag === "california") {
regiontag = "US";
} else if (regiontag === "frankfurt") {
regiontag = "EU";
regiontag = "EU-2";
} else if (regiontag === "ca"){
regiontag = "CA";
}
@@ -764,11 +773,18 @@ useEffect(() => {
"UK": "gb"
};
region = regionMapping[region_url] || "gb";
region = regionMapping[region_url] || "eu";
return `https://flagcdn.com/48x36/${region}.png`;
};
useEffect(() => {
if (window?.location?.pathname?.includes("/workflows/")) {
setExpandLeftNav(false);
}
}, [window?.location?.pathname]);
return (
<div
style={{
@@ -785,6 +801,18 @@ useEffect(() => {
zoom: 0.8,
height: "calc((100vh - 32px)*1.2)",
}}
onMouseLeave={() => {
if (window?.location?.pathname?.includes("/workflows/")) {
setExpandLeftNav(false);
}
}}
onMouseOver={() => {
if (window?.location?.pathname?.includes("/workflows/")) {
setExpandLeftNav(true);
}
}
}
>
{modalView}
<Box
@@ -905,7 +933,7 @@ useEffect(() => {
whiteSpace: "nowrap",
}}
>
<kbd>Ctrl</kbd>/<kbd>Cmd</kbd>+<kbd>K</kbd>
<kbd>Ctrl</kbd>+<kbd>K</kbd>
</span>
),
disableUnderline: true,
@@ -1153,17 +1181,14 @@ useEffect(() => {
>
<span style={{ display: "inline-block", width: "100%" }}>
<Link
to={userdata?.support ? "/dashboards/security" : "#"}
to={"#"}
style={{
...hrefStyle,
pointerEvents: userdata?.support ? "auto" : "none",
cursor: "not-allowed",
pointerEvents: "auto",
}}
>
<Button
disabled={true}
onClick={(event) => {
if (!userdata?.support) return;
setOpenSecurityTab(true);
setOpenautomateTab(false);
setCurrentOpenTab("security");
@@ -1171,21 +1196,12 @@ useEffect(() => {
}}
style={{
...ButtonStyle,
backgroundColor:
((currentOpenTab === "security" && currentPath.includes("/security")) ||
(!expandLeftNav &&
(currentPath.includes("detections") || currentPath.includes("response"))))
? "#2f2f2f"
: "transparent",
cursor: userdata?.support ? "pointer" : "not-allowed",
}}
onMouseOver={(event)=>{
event.currentTarget.style.backgroundColor = "#2f2f2f";
}}
onMouseOut={(event)=>{
event.currentTarget.style.backgroundColor = ((currentOpenTab === "security" && currentPath.includes("/security")) ||
(!expandLeftNav &&
(currentPath.includes("detections") || currentPath.includes("response"))))
event.currentTarget.style.backgroundColor = currentOpenTab === "security"
? "#2f2f2f"
: "transparent";
}}
@@ -1203,14 +1219,12 @@ useEffect(() => {
display: expandLeftNav ? "inline" : "none",
marginRight: "auto",
fontSize: 18,
color: userdata?.support
? currentOpenTab === "security" && currentPath.includes("/security")
color: currentOpenTab === "security"
? "#F1F1F1"
: "#C8C8C8"
: "#6F6F6F",
}}
>
Content
Discover
</span>
</Button>
</Link>
@@ -1452,6 +1466,8 @@ useEffect(() => {
</Button>
</Link>
</Box>
{recentworkflows?.length > 0 ?
<Box
style={{
display: "flex",
@@ -1496,6 +1512,8 @@ useEffect(() => {
}) }
</Box>
</Box>
: null }
</Box>
<Box
style={{
+10 -4
View File
@@ -1484,6 +1484,14 @@ const ParsedAction = (props) => {
const selectedAppIcon = selectedAction.large_image
var newAppname = selectedAction?.name?.charAt(0).toUpperCase() + selectedAction?.name?.substring(1)
if (newAppname === undefined || newAppname === null) {
newAppname = ""
} else {
newAppname = newAppname.replaceAll("_", " ")
}
return (
<div style={appApiViewStyle} id="parsed_action_view">
@@ -1506,11 +1514,9 @@ const ParsedAction = (props) => {
border: "2px solid rgba(255,255,255,0.3)",
}} />
</Tooltip>
<h3 style={{ }}>
{(
selectedAction?.app_name?.charAt(0).toUpperCase() +
selectedAction?.app_name?.substring(1)
)?.replaceAll("_", " ")}
{newAppname}
</h3>
</div>
<div style={{display: "flex", marginTop: 0, }}>
+34 -47
View File
@@ -409,8 +409,8 @@ const NotificationItem = memo((props) => {
{data.read === false ?
<Chip
label={"Unread"}
variant="outlined"
color="primary"
variant="contained"
color="secondary"
style={{marginRight: 15, height: 25, }}
/>
:
@@ -435,15 +435,12 @@ const NotificationItem = memo((props) => {
{data.description}
</Typography >
<div style={{ display: "flex" }}>
<ButtonGroup style={{marginTop: 15, minHeight: 50, maxHeight: 50, }}>
<Button
style={{
marginTop: 15,
textTransform: "none",
fontSize: 16,
marginRight: 5,
border: "1px solid #ff8544",
color: "#ff8544",
width: 130,
opacity: data.reference_url ? 1 : 0.5,
cursor: data.reference_url ? "pointer" : "not-allowed",
}}
@@ -459,54 +456,44 @@ const NotificationItem = memo((props) => {
{data.read === false ? (
<Button
sx={{
marginTop: "15px",
border: "none",
textTransform: "none",
fontSize: 16,
color: "#ffffff",
marginRight: "5px",
backgroundColor: "transparent",
"&:hover": { backgroundColor: "transparent" },
}}
variant="outlined"
color="secondary"
style={{
height: 50,
}}
onClick={() => {
dismissNotification(data.id);
}}
>
Dismiss
Mark Read
</Button>
) : null}
<Tooltip
title="Disabling a notification makes it so similar notifications to this one will NOT be re-opened. It will NOT forward notifications to your notification workflow, but WILL still keep counting."
placement="top"
>
<Button
style={{
marginTop: 15,
textTransform: "none",
fontSize: 16,
width: 130,
border: data.ignored
? "1px solid #2bc07e"
: "1px solid #ff8544",
color: data.ignored ? "#ffffff" : "#ff8544",
backgroundColor: data.ignored ? "#2bc07e" : "transparent",
opacity: data.ignored || data.dismissable ? 1 : 0.5,
cursor: data.dismissable ? "pointer" : "not-allowed",
}}
disabled={!data.dismissable}
onClick={() => {
if (data.ignored) {
dismissNotification(data.id, false);
} else {
dismissNotification(data.id, true);
}
}}
>
{data.ignored ? "Re-enable" : "Disable"}
</Button>
</Tooltip>
<Tooltip
title="Disabling a notification makes it so similar notifications to this one will NOT be re-opened. It will NOT forward notifications to your notification workflow, but WILL still keep counting."
placement="top"
>
<Button
variant="outlined"
color="secondary"
style={{
textTransform: "none",
opacity: data.ignored || data.dismissable ? 1 : 0.5,
cursor: data.dismissable ? "pointer" : "not-allowed",
}}
disabled={!data.dismissable}
onClick={() => {
if (data.ignored) {
dismissNotification(data.id, false);
} else {
dismissNotification(data.id, true);
}
}}
>
{data.ignored ? "Re-enable" : "Disable"}
</Button>
</Tooltip>
</ButtonGroup>
<Typography
variant="body2"
File diff suppressed because it is too large Load Diff
+8 -1
View File
@@ -3,11 +3,18 @@ export const Context = createContext();
export const AppContext =(props) => {
const currentLocation = window?.location?.pathname;
// Left side bar global states
const [searchBarModalOpen, setSearchBarModalOpen] = useState(false);
const [leftSideBarOpenByClick, setLeftSideBarOpenByClick] = useState(false);
const [leftSideBarOpenByClick, setLeftSideBarOpenByClick] = useState(currentLocation?.includes('/workflows/') ? false : true)
const [windowWidth, setWindowWidth] = useState(window.innerWidth);
useEffect(() => {
if (currentLocation?.includes('/workflows/') && leftSideBarOpenByClick === true) {
setLeftSideBarOpenByClick(false)
}
}, [leftSideBarOpenByClick])
//Calculate window width
useEffect(() => {
+16 -17
View File
@@ -12,18 +12,18 @@ const Admin2 = (props) => {
const [orgRequest, setOrgRequest] = React.useState(true);
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io";
const handleGetOrg = (orgId) => {
if (
serverside !== true &&
window.location.search !== undefined &&
window.location.search !== null
) {
const urlSearchParams = new URLSearchParams(window.location.search);
const params = Object.fromEntries(urlSearchParams.entries());
const foundorgid = params["org_id"];
if (foundorgid !== undefined && foundorgid !== null) {
orgId = foundorgid;
}
}
// if (
// serverside !== true &&
// window.location.search !== undefined &&
// window.location.search !== null
// ) {
// const urlSearchParams = new URLSearchParams(window.location.search);
// const params = Object.fromEntries(urlSearchParams.entries());
// const foundorgid = params["org_id"];
// if (foundorgid !== undefined && foundorgid !== null) {
// orgId = foundorgid;
// }
// }
console.log("getting organization details for: ", orgId);
// if (orgId === undefined) {
@@ -154,16 +154,15 @@ const Admin2 = (props) => {
});
};
const urlSearchParams = new URLSearchParams(window.location.search);
const params = Object.fromEntries(urlSearchParams.entries());
const foundOrgID = params["org_id"]
useEffect(() => {
const urlSearchParams = new URLSearchParams(window.location.search);
const params = Object.fromEntries(urlSearchParams.entries());
const foundOrgID = params["org_id"]
if(foundOrgID !== null && foundOrgID !== undefined && userdata?.support && foundOrgID?.length > 0) {
handleClickChangeOrg(foundOrgID)
}
}, [foundOrgID]);
}, [userdata]);
const handleClickChangeOrg = (orgId) => {
// Don't really care about the logout
+28 -12
View File
@@ -399,7 +399,7 @@ const AngularWorkflow = (defaultprops) => {
props.match = {}
props.match.params = params
const { leftSideBarOpenByClick, windowWidth } = useContext(Context)
const { setLeftSideBarOpenByClick, leftSideBarOpenByClick, windowWidth } = useContext(Context)
const [workflowAsCode, setWorkflowAsCode] = useState(false);
var to_be_copied = "";
@@ -8662,6 +8662,9 @@ const releaseToConnectLabel = "Release to Connect"
getApps()
fetchUsecases()
setLeftSideBarOpenByClick(false)
localStorage.setItem("expandLeftNav", false)
const cursearch = typeof window === "undefined" || window.location === undefined ? "" : window.location.search;
// FIXME: Don't check specific one here
@@ -10018,7 +10021,14 @@ const releaseToConnectLabel = "Release to Connect"
const small = props.small
const actionString = props.action
const [hover, setHover] = React.useState(false);
React.useEffect(() => {
if(app.name === "Shuffle Tools"){
if (app.actions !== undefined && (app.actions === null || app.actions.length === 1)) {
loadAppConfig(app.id, false)
}
}
}, [])
if (app === undefined || app === null) {
return (
<img
@@ -10116,6 +10126,8 @@ const releaseToConnectLabel = "Release to Connect"
}}
key={app.id}
dragging={false}
position={{x: 0, y: 0}}
defaultPosition={{x: 0, y: 0}}
>
<Paper
square
@@ -10530,6 +10542,7 @@ const releaseToConnectLabel = "Release to Connect"
x: 0,
y: 0,
}}
defaultPosition={{x: 0, y: 0}}
>
<div style={{ textDecoration: "none", color: "white", }} onClick={(event) => {
clickedApp(hit)
@@ -10568,6 +10581,9 @@ const releaseToConnectLabel = "Release to Connect"
const CustomAppHits = connectHits(AppHits)
var shuffleToolsApp = apps.find((app) => app.name === "Shuffle Tools")
if (shuffleToolsApp !== undefined && shuffleToolsApp !== null) {
shuffleToolsApp = JSON.parse(JSON.stringify(shuffleToolsApp))
}
var viewedApps = []
return (
@@ -10620,24 +10636,24 @@ const releaseToConnectLabel = "Release to Connect"
<div style={{
display: "flex",
}}>
<ParsedAppPaper small={true} action={"repeat_back_to_me"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"repeat_back_to_me"} app={shuffleToolsApp} />
<div style={{marginLeft: 5, }}/>
<ParsedAppPaper small={true} action={"filter_list"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"filter_list"} app={shuffleToolsApp} />
<div style={{marginLeft: 5, }}/>
<ParsedAppPaper small={true} action={"execute_python"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"execute_python"} app={shuffleToolsApp} />
<div style={{marginLeft: 5, }}/>
<ParsedAppPaper small={true} action={"parse_ioc"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"parse_ioc"} app={shuffleToolsApp} />
</div>
<div style={{
display: "flex",
}}>
<ParsedAppPaper small={true} action={"set_cache_value"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"set_cache_value"} app={shuffleToolsApp} />
<div style={{marginLeft: 5, }}/>
<ParsedAppPaper small={true} action={"get_file_meta"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"get_file_meta"} app={shuffleToolsApp} />
<div style={{marginLeft: 5, }}/>
<ParsedAppPaper small={true} action={"merge_lists"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"merge_lists"} app={shuffleToolsApp} />
<div style={{marginLeft: 5, }}/>
<ParsedAppPaper small={true} action={"send_sms_shuffle"} app={JSON.parse(JSON.stringify(shuffleToolsApp))} />
<ParsedAppPaper small={true} action={"send_sms_shuffle"} app={shuffleToolsApp} />
</div>
</div>
: null}
@@ -16242,7 +16258,7 @@ const releaseToConnectLabel = "Release to Connect"
{!distributedFromParent ?
isCorrectOrg ? null :
<Typography variant="body2">
<Typography variant="body2" style={{marginLeft: 10, }}>
<b>Warning</b>: Change <span
style={{color: "#FF8544", cursor: "pointer", pointerEvents: "auto", }}
onClick={() => {
@@ -16313,7 +16329,7 @@ const releaseToConnectLabel = "Release to Connect"
>Active Organization</span> to edit this Workflow.
</Typography>
:
<Typography variant="body2" color="textSecondary">
<Typography variant="body2" color="textSecondary" style={{marginLeft: 10, }}>
Warning: This workflow is controlled by your parent org and may not be editable.
</Typography>
}
+14 -7
View File
@@ -69,6 +69,7 @@ const ApiExplorerWrapper = (props) => {
const [authenticationType, setAuthenticationType] = React.useState("");
const [appAuthentication, setAppAuthentication] = useState([]);
const [selectedMeta, setSelectedMeta] = useState(undefined);
const [appLoaded, setAppLoaded] = useState(false);
const [selectedAction, setSelectedAction] = useState(
{
"app_name": selectedAppData.name,
@@ -258,6 +259,7 @@ const ApiExplorerWrapper = (props) => {
parsedapp.body === undefined ? parsedapp : JSON.parse(parsedapp.body);
setOpenapi(data);
setAppLoaded(true);
};
const handleAppAuthenticationType = (selectedAppData) => {
@@ -773,7 +775,7 @@ const ApiExplorerWrapper = (props) => {
);
};
const skeletonLoader = (
const SkeletonLoader = () => (
<Box
sx={{
display: "flex",
@@ -804,7 +806,7 @@ const ApiExplorerWrapper = (props) => {
<Skeleton variant="text" width="100%" height={40} />
</Stack>
</Box>
<Box
sx={{
display: "flex",
@@ -829,21 +831,21 @@ const ApiExplorerWrapper = (props) => {
}}
>
<Skeleton variant="text" width="100%" height={40} />
<Skeleton
variant="text"
width={100}
height={20}
sx={{ marginTop: 1 }}
/>
<Skeleton
variant="rectangular"
width="100%"
height={30}
sx={{ borderRadius: "8px", marginTop: 1 }}
/>
<Stack
spacing={2}
direction="row"
@@ -858,7 +860,7 @@ const ApiExplorerWrapper = (props) => {
<Skeleton variant="text" width={50} height={30} />
<Skeleton variant="text" width={50} height={30} />
</Stack>
<Skeleton
variant="rectangular"
width="100%"
@@ -891,6 +893,7 @@ const ApiExplorerWrapper = (props) => {
</Box>
</Box>
);
const AuthenticationData = (props) => {
const selectedApp = props.app;
@@ -1756,7 +1759,10 @@ const ApiExplorerWrapper = (props) => {
return (
<Wrapper isLoggedIn={isLoggedIn} isLoaded={isLoaded}>
<Suspense fallback={skeletonLoader}>
{appLoaded === false ? (
<SkeletonLoader />
) : (
<Suspense fallback={SkeletonLoader}>
{authenticationModal}
<ApiExplorer
openapi={openapi}
@@ -1770,6 +1776,7 @@ const ApiExplorerWrapper = (props) => {
isLoaded={isLoaded}
/>
</Suspense>
)}
</Wrapper>
);
};
+12 -7
View File
@@ -64,6 +64,7 @@ const inputColor = "#383B40";
const bodyDivStyle = {
margin: "auto",
width: "900px",
zoom: 0.8,
};
const actionListStyle = {
@@ -72,10 +73,11 @@ const actionListStyle = {
paddingBottom: "10px",
paddingTop: "10px",
marginTop: "5px",
backgroundColor: inputColor,
display: "flex",
color: "white",
position: "relative",
position: "relative",
backgroundColor: theme.palette.platformColor,
};
const boxStyle = {
@@ -89,7 +91,8 @@ const boxStyle = {
paddingTop: "30px",
display: "flex",
flexDirection: "column",
backgroundColor: surfaceColor,
backgroundColor: theme.palette.backgroundColor,
};
const dividerStyle = {
@@ -5808,9 +5811,11 @@ const AppCreator = (defaultprops) => {
variant="outlined"
color="secondary"
onClick={() => {
var urlParams = new URLSearchParams(window.location.search);
if (!urlParams.has("id")) {
window.open(`/apis/${app.id}`, "_blank")
var urlParams = new URLSearchParams(window.location.search)
if (urlParams.has("id")) {
window.open(`/apis/${urlParams.get("id")}`, "_blank")
} else if (props.match.params.appid !== undefined && props.match.params.appid !== null && props.match.params.appid.length > 0) {
window.open(`/apis/${props.match.params.appid}`, "_blank")
} else {
toast.error("Build the app first.")
}
@@ -6191,7 +6196,7 @@ const AppCreator = (defaultprops) => {
{testView}
*/}
<div style={{height: 50, padding: 25, display: "flex", marginTop: 35, position: "fixed", bottom: 0, left: 0, width: "100%", backgroundColor: theme.palette?.backgroundColor, borderTop: "1px solid rgba(255,255,255,0.3)",}}>
<div style={{height: 50, padding: 15, display: "flex", marginTop: 35, position: "fixed", bottom: 0, left: 0, width: "100%", backgroundColor: theme.palette?.backgroundColor, borderTop: "1px solid rgba(255,255,255,0.3)",}}>
<div style={{width: 450, margin: "auto", display: "flex", textAlign: "center", }}>
{appDownloadData.length > 0 ?
<Tooltip title="Download the OpenAPI specification for the App" placement="bottom">
File diff suppressed because it is too large Load Diff
+29 -18
View File
@@ -716,6 +716,8 @@ const Workflows2 = (props) => {
const [videoViewOpen, setVideoViewOpen] = React.useState(false)
const [gettingStartedItems, setGettingStartedItems] = React.useState([])
const [selectedWorkflowIndexes, setSelectedWorkflowIndexes] = React.useState([])
const [page, setPage] = React.useState(0);
const [pageSize, setPageSize] = React.useState(100);
const [highlightIds, setHighlightIds] = React.useState([])
const [apps, setApps] = React.useState([]);
@@ -751,15 +753,6 @@ const Workflows2 = (props) => {
navigate(`${location.pathname}?${queryParams.toString()}`);
};
useEffect(() => {
if (currTab === 2) {
setIsLoadingPublicWorkflow(true);
// Simulate loading time for the Algolia search results
setTimeout(() => {
setIsLoadingPublicWorkflow(false);
}, 2500);
}
}, [currTab])
@@ -1319,8 +1312,17 @@ const Workflows2 = (props) => {
return response.json();
})
.then((responseJson) => {
console.log("Response : /api/v1/workflows", responseJson)
if (responseJson !== undefined) {
if (responseJson !== undefined && responseJson !== null) {
if (responseJson.success === false) {
} else if (responseJson.length === 0) {
// When there are no workflows, we can set the loading to false
setIsLoadingWorkflow(false)
if (currTab !== 2) {
toast("No workflows found. Showing workflow discovery")
setCurrTab(2)
}
}
var newarray = []
for (var wfkey in responseJson) {
const wf = responseJson[wfkey]
@@ -2476,6 +2478,7 @@ const Workflows2 = (props) => {
}
}
return (
<div style={{ width: "100%", minWidth: 320, position: "relative", border: highlightIds.includes(data.id) ? "2px solid #f85a3e" : isDistributed || hasSuborgs ? "2px solid #40E0D0" : "inherit", borderRadius: theme.palette?.borderRadius, backgroundColor: "#212121", fontFamily: theme?.typography?.fontFamily }}>
<Paper square style={paperAppStyle}>
@@ -2530,7 +2533,7 @@ const Workflows2 = (props) => {
maxWidth: 310,
padding: "12px 0",
}}>
{(data?.image !== undefined || data?.image_url !== undefined) ? (
{(data?.image !== undefined || (data?.image_url !== undefined && data?.image_url.length > 0)) ? (
<div style={{
marginBottom: 15,
borderRadius: theme.palette?.borderRadius,
@@ -3364,7 +3367,15 @@ const Workflows2 = (props) => {
className={classes.datagrid}
rows={rows}
columns={columns}
pageSize={100}
page={page}
onPageChange={(newPage) => {
setPage(newPage)
}}
pageSize={pageSize}
onPageSizeChange={(newPageSize) => {
setPageSize(newPageSize);
}}
rowsPerPageOptions={[25, 50, 100, 150]}
checkboxSelection
autoHeight
density="standard"
@@ -3955,7 +3966,7 @@ const Workflows2 = (props) => {
setIsLoadingWorkflow(false);
}
}, [currTab, workflows, userdata, filteredWorkflows])
}, [currTab, workflows, userdata, filteredWorkflows, filters])
@@ -4243,7 +4254,7 @@ const Workflows2 = (props) => {
margin: "auto",
}}>
<Typography variant="h4" style={{ marginBottom: 20, paddingLeft: 15, textTransform: 'none', fontFamily: theme?.typography?.fontFamily }}>
Workflows
{currTab === 0 ? "Org" : currTab === 1 ? "Your" : "Discover"} Workflows
</Typography>
<div style={{ borderBottom: '1px solid gray', marginBottom: 30 }}>
@@ -4258,21 +4269,21 @@ const Workflows2 = (props) => {
TabIndicatorProps={{ style: { display: 'none' } }}
>
<Tab
label="Organization Apps"
label="Organization Workflows"
style={{
...tabStyle,
...(currTab === 0 ? tabActive : {})
}}
/>
<Tab
label="My Apps"
label="My Workflows"
style={{
...tabStyle,
...(currTab === 1 ? tabActive : {})
}}
/>
<Tab
label="Discover Apps"
label="Discover Workflows"
style={{
...tabStyle,
marginRight: 0,
+20 -27
View File
@@ -1,36 +1,29 @@
FROM golang:1.22 as builder
RUN mkdir /app
WORKDIR /app
COPY orborus.go /app/orborus.go
#COPY go.mod /app/go.mod
#COPY go.sum /app/go.sum
RUN go mod init orborus
# COPY go.* ./
RUN go mod init orborus
RUN go get github.com/docker/docker/api/types
RUN go get github.com/docker/docker/api/types/container
#RUN go get github.com/docker/docker/client
RUN go get github.com/mackerelio/go-osstat/cpu
RUN go get github.com/mackerelio/go-osstat/memory
RUN go get github.com/satori/go.uuid
RUN go get github.com/shuffle/shuffle-shared
RUN go get github.com/shirou/gopsutil
RUN go get k8s.io/client-go
RUN go get k8s.io/apimachinery
RUN go get \
github.com/docker/docker/api/types \
github.com/docker/docker/api/types/container \
github.com/mackerelio/go-osstat/cpu \
github.com/mackerelio/go-osstat/memory \
github.com/satori/go.uuid \
github.com/shuffle/shuffle-shared \
github.com/shirou/gopsutil
RUN go get
RUN go mod tidy
RUN go build
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
COPY orborus.go .
RUN go mod tidy && \
CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
FROM alpine:3.15.0
RUN apk add --no-cache bash tzdata
COPY --from=builder /app/ /
ENV ENVIRONMENT_NAME=Shuffle
ENV BASE_URL=http://shuffle-backend:5001
ENV DOCKER_API_VERSION=1.40
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
CMD ["./orborus"]
COPY --from=builder /app/orborus /
ENV ENVIRONMENT_NAME=Shuffle \
BASE_URL=http://shuffle-backend:5001 \
DOCKER_API_VERSION=1.40 \
SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
CMD ["./orborus"]
+36
View File
@@ -0,0 +1,36 @@
FROM golang:1.22 as builder
RUN mkdir /app
WORKDIR /app
COPY orborus.go /app/orborus.go
#COPY go.mod /app/go.mod
#COPY go.sum /app/go.sum
RUN go mod init orborus
RUN go get github.com/docker/docker/api/types
RUN go get github.com/docker/docker/api/types/container
#RUN go get github.com/docker/docker/client
RUN go get github.com/mackerelio/go-osstat/cpu
RUN go get github.com/mackerelio/go-osstat/memory
RUN go get github.com/satori/go.uuid
RUN go get github.com/shuffle/shuffle-shared
RUN go get github.com/shirou/gopsutil
# RUN go get k8s.io/client-go
# RUN go get k8s.io/apimachinery
RUN go get
RUN go mod tidy
RUN go build
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
FROM alpine:3.15.0
RUN apk add --no-cache bash tzdata
COPY --from=builder /app/ /
ENV ENVIRONMENT_NAME=Shuffle
ENV BASE_URL=http://shuffle-backend:5001
ENV DOCKER_API_VERSION=1.40
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
CMD ["./orborus"]
+6 -6
View File
@@ -10,7 +10,7 @@ require (
github.com/docker/docker v27.0.2+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.6.90
github.com/shuffle/shuffle-shared v0.6.93
k8s.io/api v0.30.2
k8s.io/apimachinery v0.30.2
)
@@ -95,14 +95,14 @@ require (
go.opentelemetry.io/otel/sdk v1.30.0 // indirect
go.opentelemetry.io/otel/trace v1.30.0 // indirect
go4.org v0.0.0-20201209231011-d4a079459e60 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/term v0.24.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+10 -10
View File
@@ -366,8 +366,8 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -443,8 +443,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -473,16 +473,16 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -494,8 +494,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+23 -11
View File
@@ -199,7 +199,9 @@ func skipCheckInCleanup(name string) bool {
strings.HasPrefix(name, "orborus") ||
strings.HasPrefix(name, "shuffle-orborus") ||
strings.HasPrefix(name, "opensearch") ||
strings.HasPrefix(name, "shuffle-opensearch")
strings.HasPrefix(name, "shuffle-opensearch") ||
strings.HasPrefix(name, "memcached") ||
strings.HasPrefix(name, "shuffle-memcached")
}
func cleanupExistingNodes(ctx context.Context) error {
@@ -608,8 +610,7 @@ func deployServiceWorkers(image string) {
Condition: swarm.RestartPolicyConditionOnFailure,
},
Placement: &swarm.Placement{
Constraints: []string{
},
Constraints: []string{},
},
},
}
@@ -2152,7 +2153,7 @@ func main() {
}
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" && os.Getenv("SHUFFLE_SCALE_REPLICAS") == "" {
go AutoScale(ctx)
// go AutoScale(ctx)
}
hasStarted = true
}
@@ -2297,11 +2298,20 @@ func main() {
} else if incRequest.Type == "START_TENZIR" {
log.Printf("[INFO] Got job to start tenzir")
// Manual command = overrides to allow starting of Tenzir from the frontend anyway.
os.Setenv("SHUFFLE_SKIP_PIPELINES", "false")
tenzirDisabled = false
// Removed either way
toBeRemoved.Data = append(toBeRemoved.Data, incRequest)
err := deployTenzirNode()
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "node available") {
toBeRemoved.Data = append(toBeRemoved.Data, incRequest)
} else {
// Disabling until UI is updated
os.Setenv("SHUFFLE_SKIP_PIPELINES", "true")
tenzirDisabled = true
log.Printf("[ERROR] Failed to start tenzir, reason: %s", err)
err = shuffle.CreateOrgNotification(
ctx,
@@ -2317,9 +2327,6 @@ func main() {
return
}
}
} else {
toBeRemoved.Data = append(toBeRemoved.Data, incRequest)
}
} else {
@@ -2620,7 +2627,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error {
func deployTenzirNode() error {
if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" {
// return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES")
log.Printf("[INFO] Pipelines are enabled by user")
//log.Printf("[INFO] Pipelines are enabled by user")
} else {
return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES")
}
@@ -2637,7 +2644,7 @@ func deployTenzirNode() error {
ctx := context.Background()
cacheKey := "tenzir-key"
imageName := "tenzir/tenzir:latest"
imageName := "frikky/shuffle:tenzir"
containerName := "tenzir-node"
containerStartOptions := container.StartOptions{}
_, err = shuffle.GetCache(ctx, cacheKey)
@@ -2822,9 +2829,13 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
},
}
// FIXME: Is this necessary? Seems to screw up networking:
// conflicting options: hostname and the network mode
/*
if isKubernetes != "true" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" {
hostConfig.NetworkMode = container.NetworkMode(fmt.Sprintf("container:%s", containerId))
}
*/
resp, err := dockercli.ContainerCreate(ctx, config, hostConfig, networkingConfig, nil, containerName)
if err != nil {
@@ -3455,6 +3466,7 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) {
if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") {
log.Printf("[ERROR] Tenzir node connection problem: %s", err)
} else {
tenzirDisabled = true
log.Printf("[ERROR] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
+38 -45
View File
@@ -27,13 +27,13 @@ import (
dockerclient "github.com/docker/docker/client"
// This is for automatic removal of certain code :)
/*** ***/
/*** STARTREMOVE ***/
"math/rand"
"github.com/docker/docker/api/types/swarm"
uuid "github.com/satori/go.uuid"
/*** ***/
/*** ENDREMOVE ***/
"github.com/gorilla/mux"
@@ -109,7 +109,7 @@ var autoDeploy = map[string]string{
"shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0",
"shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0",
"shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0",
"shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0",
// "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0",
}
//"testing:1.0.0": "frikky/shuffle:testing_1.0.0",
@@ -321,11 +321,11 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
*/
} else {
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" {
log.Printf("[DEBUG][%s] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", workflowExecution.ExecutionId, 0, cleanupEnv)
}
/*** ***/
/*** ENDREMOVE ***/
}
if len(reason) > 0 && len(nodeId) > 0 {
@@ -393,7 +393,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
func int32Ptr(i int32) *int32 { return &i }
// ** ***/
// ** STARTREMOVE ***/
func deployk8sApp(image string, identifier string, env []string) error {
if len(os.Getenv("KUBERNETES_NAMESPACE")) > 0 {
kubernetesNamespace = os.Getenv("KUBERNETES_NAMESPACE")
@@ -631,7 +631,7 @@ func deployk8sApp(image string, identifier string, env []string) error {
return nil
}
//** ***/
//** ENDREMOVE ***/
// Deploys the internal worker whenever something happens
func deployApp(cli *dockerclient.Client, image string, identifier string, env []string, workflowExecution shuffle.WorkflowExecution, action shuffle.Action) error {
@@ -772,7 +772,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
}
}
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" {
appName := strings.Replace(identifier, fmt.Sprintf("_%s", action.ID), "", -1)
@@ -802,14 +802,14 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
return err
}
} else {
// ** ***/
// ** STARTREMOVE ***/
exposedPort = 80
err = findAppInfoKubernetes(image, appName, env)
if err != nil {
log.Printf("[ERROR] Failed finding and creating port for %s: %s", appName, err)
return err
}
// ** ***/
// ** ENDREMOVE ***/
}
/*
@@ -852,7 +852,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env []
return nil
}
/*** ***/
/*** ENDREMOVE ***/
// Max 10% CPU every second
//CPUShares: 128,
@@ -2404,7 +2404,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
return
}
/*** ***/
/*** STARTREMOVE ***/
if workflowExecution.Status == "WAITING" && (os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm") {
log.Printf("[INFO][%s] Workflow execution is waiting while in swarm. Sending info to backend to ensure execution stops.", workflowExecution.ExecutionId)
@@ -2416,7 +2416,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
shutdown(*workflowExecution, "", "", false)
}
}
/*** ***/
/*** ENDREMOVE ***/
} else {
if strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already been ran") || strings.Contains(strings.ToLower(fmt.Sprintf("%s", err)), "already finished") {
log.Printf("[ERROR][%s] Skipping rerun of action result as it's already been ran: %s", workflowExecution.ExecutionId)
@@ -2501,7 +2501,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
return
}
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" {
finished := shuffle.ValidateFinished(ctx, -1, *workflowExecution)
if !finished {
@@ -2516,7 +2516,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
sendResult(*workflowExecution, shutdownData)
}
}
/*** ***/
/*** ENDREMOVE ***/
} else {
log.Printf("[INFO][%s] Skipping setexec with status %s", workflowExecution.ExecutionId, workflowExecution.Status)
@@ -2536,12 +2536,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl
func sendSelfRequest(actionResult shuffle.ActionResult) {
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" {
log.Printf("[INFO][%s] Not sending self request info since source is default (not swarm)", actionResult.ExecutionId)
return
}
/*** ***/
/*** ENDREMOVE ***/
data, err := json.Marshal(actionResult)
if err != nil {
@@ -2680,11 +2680,11 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool {
}
}
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" {
requestsSent += 1
}
/*** ***/
/*** ENDREMOVE ***/
log.Printf("[DEBUG][%s] Should send full result to %s", workflowExecution.ExecutionId, baseUrl)
@@ -2769,7 +2769,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) {
// GetLocalIP returns the non loopback local IP of the host
func getLocalIP() string {
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("IS_KUBERNETES") == "true" {
return "shuffle-workers"
}
@@ -2836,7 +2836,7 @@ func getLocalIP() string {
return foundIP
}
}
/*** ***/
/*** ENDREMOVE ***/
addrs, err := net.InterfaceAddrs()
if err != nil {
@@ -2887,7 +2887,7 @@ func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener {
log.Printf("[DEBUG] OLD HOSTNAME: %s", appCallbackUrl)
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" {
log.Printf("[DEBUG] Starting webserver (1) on port %d with hostname: %s", baseport, hostname)
@@ -2907,7 +2907,7 @@ func webserverSetup(workflowExecution shuffle.WorkflowExecution) net.Listener {
return listener
}
/*** ***/
/*** ENDREMOVE ***/
port := listener.Addr().(*net.TCPAddr).Port
// Set the port environment variable
@@ -2958,7 +2958,7 @@ func findActiveSwarmNodes(dockercli *dockerclient.Client) (int64, error) {
*/
}
/*** ***/
/*** STARTREMOVE ***/
func deploySwarmService(dockercli *dockerclient.Client, name, image string, deployport int) error {
log.Printf("[DEBUG] Deploying service for %s to swarm on port %d", name, deployport)
//containerName := fmt.Sprintf("shuffle-worker-%s", parsedUuid)
@@ -3128,10 +3128,10 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
return nil
}
/*** ***/
/*** ENDREMOVE ***/
// Runs data discovery
/*** ***/
/*** STARTREMOVE ***/
func findAppInfoKubernetes(image, name string, env []string) error {
clientset, _, err := shuffle.GetKubernetesClient()
@@ -3291,7 +3291,7 @@ func findAppInfo(image, name string) (int, error) {
return exposedPort, nil
}
/*** ***/
/*** ENDREMOVE ***/
func sendAppRequest(ctx context.Context, incomingUrl, appName string, port int, action *shuffle.Action, workflowExecution *shuffle.WorkflowExecution) error {
parsedRequest := shuffle.OrborusExecutionRequest{
@@ -3691,11 +3691,11 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe
// Initial loop etc
func main() {
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" {
logsDisabled = "true"
}
/*** ***/
/*** ENDREMOVE ***/
// Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes
// Recursive import kind of :)
_, err := shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), "", "worker", true, "elasticsearch", false, 0)
@@ -3729,7 +3729,7 @@ func main() {
swarmConfig := os.Getenv("SHUFFLE_SWARM_CONFIG")
log.Printf("[INFO] Running with timezone %s and swarm config %#v", timezone, swarmConfig)
/*** ***/
/*** STARTREMOVE ***/
if swarmConfig == "run" || swarmConfig == "swarm" {
// Forcing download just in case on the first iteration.
log.Printf("[INFO] Running in swarm mode - forcing download of apps")
@@ -3744,7 +3744,7 @@ func main() {
log.Printf("[ERROR] Stopped listener %#v - exiting.", listener)
os.Exit(3)
}
/*** ***/
/*** ENDREMOVE ***/
authorization := ""
executionId := ""
@@ -4097,14 +4097,13 @@ func runWebserver(listener net.Listener) {
// What would be require to run a workflow otherwise?
// Maybe directly /workflow/run
/*** ***/
/*** STARTREMOVE ***/
if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || os.Getenv("SHUFFLE_SWARM_CONFIG") == "swarm" {
log.Printf("[DEBUG] Running webserver config for SWARM and K8s")
}
/*** ***/
var dockercli *dockerclient.Client
defer dockercli.Close()
/*** ENDREMOVE ***/
// var dockercli *dockerclient.Client
// ctx := context.Background()
scaleReplicas := os.Getenv("SHUFFLE_APP_REPLICAS")
if len(scaleReplicas) > 0 {
tmpInt, err := strconv.Atoi(scaleReplicas)
@@ -4130,13 +4129,9 @@ func runWebserver(listener net.Listener) {
log.Printf("[DEBUG] SHUFFLE_APP_EXECUTIONS_PER_MINUTE set to value %s. Trying to overwrite default (%d)", os.Getenv("SHUFFLE_APP_EXECUTIONS_PER_MINUTE"), maxExecutionsPerMinute)
}
/*
ctx := context.Background()
if strings.ToLower(os.Getenv("SHUFFLE_SWARM_CONFIG")) == "run" || strings.ToLower(os.Getenv("SHUFFLE_APP_REPLICAS")) == "" {
go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute)
// go AutoScaleApps(ctx, dockercli, maxExecutionsPerMinute)
}
*/
if strings.ToLower(os.Getenv("SHUFFLE_DEBUG_MEMORY")) == "true" {
r.HandleFunc("/debug/pprof/", pprof.Index)
r.HandleFunc("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
@@ -4182,14 +4177,12 @@ func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutio
case <-ticker.C:
count := window.CountEvents(time.Now())
appNumber := numberOfApps(ctx, client)
j := numberOfApps(ctx, client)
workers := numberOfWorkers(ctx, client)
execPerMin := maxExecutionsPerMinute / workers
if count >= execPerMin {
log.Printf("[DEBUG] Too many executions per minute (%d). Scaling down to %d", count, execPerMin)
scaleApps(ctx, client, uint64(appNumber+1))
scaleApps(ctx, client, uint64(j+1))
}
}
}