{selectedOrganization.name.length > 0 ?
- : null}
+ :
+
+
+
+ Loading Organization
+
+
+ }
Cloud syncronization
What does cloud sync do? Cloud syncronization is a way of getting more out of Shuffle. Shuffle will ALWAYS make every option open source, but features relying on other users can't be done without a collaborative approach.
@@ -1690,7 +1703,7 @@ const Admin = (props) => {
style={{ minWidth: 180, maxWidth: 180 }}
/>
- {users === undefined ? null : users.map((data, index) => {
+ {users === undefined || users === null ? null : users.map((data, index) => {
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
@@ -2074,7 +2087,7 @@ const Admin = (props) => {
primary="Actions"
/>
- {authentication === undefined ? null : authentication.map((data, index) => {
+ {authentication === undefined || authentication === null ? null : authentication.map((data, index) => {
var bgColor = "#27292d"
if (index % 2 === 0) {
bgColor = "#1f2023"
@@ -2392,7 +2405,7 @@ const Admin = (props) => {
const iconStyle = {marginRight: 10}
const data =
-
+
0 && strings.Contains(items[1], "(AppBase)") {
- classname = strings.Split(items[1], "(")[0]
- } else {
- log.Println("Something wrong :( (horrible programming right here)")
- os.Exit(3)
- }
- }
-
- if strings.Contains(line, "if __name__ ==") {
- break
- }
-
- // asyncio.run(HelloWorld.run(), debug=True)
-
- newfile = append(newfile, line)
- }
-
- filedata = []byte(strings.Join(newfile, "\n"))
- return classname, filedata
-}
-
-// https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
-func Copy(src, dst string) error {
- in, err := os.Open(src)
- if err != nil {
- return err
- }
- defer in.Close()
-
- out, err := os.Create(dst)
- if err != nil {
- return err
- }
- defer out.Close()
-
- _, err = io.Copy(out, in)
- if err != nil {
- return err
- }
- return out.Close()
-}
-func ZipFiles(filename string, files []string) error {
- newZipFile, err := os.Create(filename)
- if err != nil {
- return err
- }
- defer newZipFile.Close()
-
- zipWriter := zip.NewWriter(newZipFile)
- defer zipWriter.Close()
-
- // Add files to zip
- for _, file := range files {
- zipfile, err := os.Open(file)
- if err != nil {
- return err
- }
- defer zipfile.Close()
-
- // Get the file information
- info, err := zipfile.Stat()
- if err != nil {
- return err
- }
-
- header, err := zip.FileInfoHeader(info)
- if err != nil {
- return err
- }
-
- // Using FileInfoHeader() above only uses the basename of the file. If we want
- // to preserve the folder structure we can overwrite this with the full path.
- filesplit := strings.Split(file, "/")
- if len(filesplit) > 1 {
- header.Name = filesplit[len(filesplit)-1]
- } else {
- header.Name = file
- }
-
- // Change to deflate to gain better compression
- // see http://golang.org/pkg/archive/zip/#pkg-constants
- header.Method = zip.Deflate
-
- writer, err := zipWriter.CreateHeader(header)
- if err != nil {
- return err
- }
- if _, err = io.Copy(writer, zipfile); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func getAppbase(filepath string) []string {
- appBase, err := ioutil.ReadFile(filepath)
- if err != nil {
- log.Printf("Readerror: %s", err)
- os.Exit(1)
- }
-
- record := false
- validLines := []string{}
- for _, line := range strings.Split(string(appBase), "\n") {
- if strings.Contains(line, "#STOPCOPY") {
- log.Println("Stopping copy")
- break
- }
-
- if record {
- validLines = append(validLines, line)
- }
-
- if strings.Contains(line, "#STARTCOPY") {
- log.Println("Starting copy")
- record = true
- }
- }
-
- return validLines
-}
-
-// Puts together ./static_baseline.py, onprem/app_sdk_app_base.py and the
-// appcode in a generated_app folder based on appname+version
-func stitcher(appname string, appversion string) string {
- baselinefile := "static_baseline.py"
- appfolder := "apps"
- appbasefile := "onprem/app_sdk/app_base.py"
-
- baseline, err := ioutil.ReadFile(baselinefile)
- if err != nil {
- log.Printf("Readerror: %s", err)
- os.Exit(1)
- }
-
- sourceappfile := fmt.Sprintf("%s/%s/%s/src/app.py", appfolder, appname, appversion)
- appfile, err := ioutil.ReadFile(sourceappfile)
- if err != nil {
- log.Printf("App readerror: %s", err)
- os.Exit(1)
- }
-
- classname, appfile := formatAppfile(appfile)
- if len(classname) == 0 {
- log.Println("Failed finding classname in file.")
- os.Exit(3)
- }
-
- runner := getRunner(classname)
- appBase := getAppbase(appbasefile)
-
- foldername := fmt.Sprintf("generated_apps/%s_%s", appname, appversion)
- err = os.Mkdir(foldername, os.ModePerm)
- if err != nil {
- log.Println("Failed making temporary app folder. Probably already exists. Remaking")
- os.RemoveAll(foldername)
- os.MkdirAll(foldername, os.ModePerm)
- }
-
- stitched := []byte(string(baseline) + strings.Join(appBase, "\n") + string(appfile) + string(runner))
- err = ioutil.WriteFile(fmt.Sprintf("%s/main.py", foldername), stitched, os.ModePerm)
- if err != nil {
- log.Println("Failed writing to stitched: %s", err)
- os.Exit(3)
- }
-
- err = Copy(fmt.Sprintf("%s/%s/%s/requirements.txt", appfolder, appname, appversion), fmt.Sprintf("%s/requirements.txt", foldername))
- if err != nil {
- log.Println("Failed writing to requirement: %s", err)
- os.Exit(3)
- }
-
- log.Printf("Successfully stitched files in %s/main.py", foldername)
- // Zip the folder
- files := []string{
- fmt.Sprintf("%s/main.py", foldername),
- fmt.Sprintf("%s/requirements.txt", foldername),
- }
- outputfile := fmt.Sprintf("%s.zip", foldername)
-
- err = ZipFiles(outputfile, files)
- if err != nil {
- log.Fatal(err)
- }
-
- ctx := context.Background()
-
- // Creates a client.
- client, err := storage.NewClient(ctx)
- if err != nil {
- log.Printf("Failed to create client: %v", err)
- os.Exit(3)
- }
-
- // Create bucket handle
- bucket := client.Bucket(bucketName)
-
- remotePath := fmt.Sprintf("apps/%s_%s.zip", appname, appversion)
- err = createFileFromFile(bucket, remotePath, outputfile)
- if err != nil {
- log.Printf("Failed to upload to bucket: %v", err)
- os.Exit(3)
- }
-
- os.Remove(outputfile)
- return fmt.Sprintf("gs://%s/apps/%s_%s.zip", bucketName, appname, appversion)
-}
-
-func createFileFromFile(bucket *storage.BucketHandle, remotePath, localPath string) error {
- ctx := context.Background()
- // [START upload_file]
- f, err := os.Open(localPath)
- if err != nil {
- return err
- }
- defer f.Close()
-
- wc := bucket.Object(remotePath).NewWriter(ctx)
- if _, err = io.Copy(wc, f); err != nil {
- return err
- }
- if err := wc.Close(); err != nil {
- return err
- }
- // [END upload_file]
- return nil
-}
-
-// Deploy to google cloud function :)
-func deployFunction(appname, localization, applocation string, environmentVariables map[string]string) error {
- ctx := context.Background()
- service, err := cloudfunctions.NewService(ctx)
- if err != nil {
- return err
- }
-
- // ProjectsLocationsListCall
- projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service)
- location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization)
- functionName := fmt.Sprintf("%s/functions/%s", location, appname)
-
- cloudFunction := &cloudfunctions.CloudFunction{
- AvailableMemoryMb: 128,
- EntryPoint: "authorization",
- EnvironmentVariables: environmentVariables,
- HttpsTrigger: &cloudfunctions.HttpsTrigger{},
- MaxInstances: 0,
- Name: functionName,
- Runtime: "python37",
- SourceArchiveUrl: applocation,
- }
-
- //getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location))
- //resp, err := getCall.Do()
-
- createCall := projectsLocationsFunctionsService.Create(location, cloudFunction)
- _, err = createCall.Do()
- if err != nil {
- log.Println("Failed creating new function. Attempting patch, as it might exist already")
-
- createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, appname), cloudFunction)
- _, err = createCall.Do()
- if err != nil {
- log.Println("Failed patching function")
- return err
- }
-
- log.Printf("Successfully patched %s to %s", appname, localization)
- } else {
- log.Printf("Successfully deployed %s to %s", appname, localization)
- }
-
- // FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho
-
- return nil
-}
-
-func deployAppCloudFunc(appname string, appversion string) {
- _ = os.Mkdir("generated_apps", os.ModePerm)
-
- apikey := "eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
- fullAppname := fmt.Sprintf("%s-%s", strings.Replace(appname, "_", "-", -1), strings.Replace(appversion, ".", "-", -1))
- locations := []string{"europe-west2"}
-
- // Deploys the app to all locations
- bucketname := stitcher(appname, appversion)
- environmentVariables := map[string]string{
- "FUNCTION_APIKEY": apikey,
- }
-
- for _, location := range locations {
- err := deployFunction(fullAppname, location, bucketname, environmentVariables)
- if err != nil {
- log.Printf("Failed to deploy: %s", err)
- os.Exit(3)
- }
- }
-}
-
-func loadYaml(fileLocation string) (WorkflowApp, error) {
- action := WorkflowApp{}
-
- yamlFile, err := ioutil.ReadFile(fileLocation)
- if err != nil {
- log.Printf("yamlFile.Get err: %s", err)
- return WorkflowApp{}, err
- }
-
- //log.Printf(string(yamlFile))
- err = yaml.Unmarshal([]byte(yamlFile), &action)
- if err != nil {
- return WorkflowApp{}, err
- }
-
- return action, nil
-}
-
-// FIXME - deploy to backend (YAML config)
-func deployConfigToBackend(appname string, appversion string) error {
- // FIXME - no static path pls
- action, err := loadYaml(fmt.Sprintf("apps/%s/%s/api.yaml", appname, appversion))
- if err != nil {
- log.Println(err)
- return err
- }
-
- action.Sharing = true
-
- data, err := json.Marshal(action)
- if err != nil {
- return err
- }
-
- url := "http://localhost:5001/api/v1/workflows/apps"
- client := &http.Client{}
- req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
- if err != nil {
- return err
- }
-
- req.Header.Set("Authorization", "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ")
-
- ret, err := client.Do(req)
- if err != nil {
- return err
- }
-
- log.Printf("Status: %s", ret.Status)
- body, err := ioutil.ReadAll(ret.Body)
- if err != nil {
- return err
- }
-
- if ret.StatusCode != 200 {
- return errors.New(fmt.Sprintf("Status %s. App probably already exists. Raw:\n%s", ret.Status, string(body)))
- }
-
- log.Println(string(body))
- return nil
-}
-
-func tarDirectory(filecontext string) (io.Reader, error) {
-
- // Create a filereader
- //dockerFileReader, err := os.Open(dockerfile)
- //if err != nil {
- // return err
- //}
-
- //// Read the actual Dockerfile
- //readDockerFile, err := ioutil.ReadAll(dockerFileReader)
- //if err != nil {
- // return err
- //}
-
- // Make a TAR header for the file
- tarHeader := &tar.Header{
- Name: filecontext,
- Typeflag: tar.TypeDir,
- }
-
- // Writes the header described for the TAR file
- buf := new(bytes.Buffer)
- tw := tar.NewWriter(buf)
- defer tw.Close()
- err := tw.WriteHeader(tarHeader)
- if err != nil {
- return nil, err
- }
-
- dockerFileTarReader := bytes.NewReader(buf.Bytes())
- return dockerFileTarReader, nil
-}
-
-func tarDir(source string, target string) (*bytes.Reader, error) {
- filename := filepath.Base(source)
- target = filepath.Join(target, fmt.Sprintf("%s.tar", filename))
- tarfile, err := os.Create(target)
- if err != nil {
- return nil, err
- }
-
- defer tarfile.Close()
-
- buf := new(bytes.Buffer)
- _ = buf
- tarball := tar.NewWriter(tarfile)
- defer tarball.Close()
-
- info, err := os.Stat(source)
- if err != nil {
- return nil, err
- }
-
- var baseDir string
- if info.IsDir() {
- baseDir = filepath.Base(source)
- }
-
- _ = filepath.Walk(source,
- func(path string, info os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- header, err := tar.FileInfoHeader(info, info.Name())
- if err != nil {
- return err
- }
-
- if baseDir != "" {
- header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
- }
-
- if err := tarball.WriteHeader(header); err != nil {
- return err
- }
-
- if info.IsDir() {
- return nil
- }
-
- file, err := os.Open(path)
- if err != nil {
- return err
- }
- defer file.Close()
- _, err = io.Copy(tarball, file)
- return nil
- })
-
- dockerFileTarReader := bytes.NewReader(buf.Bytes())
- return dockerFileTarReader, nil
-}
-
-func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) error {
- dockerBuildContext, err := tarDir(dockerBuildCtxDir, ".")
- if err != nil {
- log.Printf("Error in taring the docker root folder - %s", err.Error())
- return err
- }
-
- imageBuildResponse, err := client.ImageBuild(
- context.Background(),
- dockerBuildContext,
- types.ImageBuildOptions{
- Dockerfile: "Dockerfile",
- PullParent: true,
- Remove: true,
- Tags: tags,
- NetworkMode: "host",
- },
- )
-
- if err != nil {
- return err
- }
-
- // Read the STDOUT from the build process
- defer imageBuildResponse.Body.Close()
- _, err = io.Copy(os.Stdout, imageBuildResponse.Body)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-// FIXME - deploy to dockerhub
-func deployWorker(appname, appversion string) error {
- // Get dockerfile from ./apps/appname/appversion/Dockerfile
- client, err := client.NewEnvClient()
- if err != nil {
- return err
- }
-
- tags := []string{fmt.Sprintf("%s-%s", appname, appversion)}
- err = buildImage(client, tags, fmt.Sprintf("./apps/%s/%s", appname, appversion))
- if err != nil {
- log.Printf("Build error: %s", err)
- return err
- }
-
- return nil
-}
-
-// Deploys all cloud functions. Onprem thooo :(
-func deployAll() {
- allapps := []string{
- "hoxhunt",
- "secureworks",
- "servicenow",
- "lastline",
- "netcraft",
- "misp",
- "email",
- "testing",
- "http",
- "recordedfuture",
- "passivetotal",
- "carbon_black",
- "thehive",
- "cortex",
- "splunk",
- }
-
- for _, appname := range allapps {
- appversion := "1.0.0"
-
- err := deployConfigToBackend(appname, appversion)
- if err != nil {
- log.Printf("Failed uploading config: %s", err)
- continue
- }
-
- deployAppCloudFunc(appname, appversion)
- }
-}
-
-func main() {
- deployAll()
- return
-
- appname := "testing"
- appversion := "1.0.0"
-
- err := deployConfigToBackend(appname, appversion)
- if err != nil {
- log.Printf("Failed uploading config: %s", err)
- os.Exit(1)
- }
-
- deployAppCloudFunc(appname, appversion)
-
- // FIXME - build and deploy to dockerhub as well :)
- // Not able to work in remote directory propely... Even tried making an actual tar and checking it rofl
- //err := deployWorker(appname, appversion)
- //if err != nil {
- // log.Printf("Failed to deploy docker worker: %s", err)
- //}
-}