package main import ( "bufio" "bytes" "context" "crypto/md5" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/ioutil" "log" "net" "net/http" "os" "os/exec" //"regexp" "strconv" "strings" "time" // Google cloud "cloud.google.com/go/datastore" "cloud.google.com/go/pubsub" "cloud.google.com/go/storage" "google.golang.org/appengine/mail" "github.com/getkin/kin-openapi/openapi2" "github.com/getkin/kin-openapi/openapi2conv" "github.com/getkin/kin-openapi/openapi3" "github.com/google/go-github/v28/github" "golang.org/x/oauth2" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/storage/memory" // Random xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" gyaml "github.com/ghodss/yaml" "github.com/satori/go.uuid" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" // Web // "github.com/gorilla/handlers" "github.com/gorilla/mux" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Old items (cloud) // "google.golang.org/appengine" // "google.golang.org/appengine/memcache" // applog "google.golang.org/appengine/log" //cloudrun "google.golang.org/api/run/v1" ) // This is used to handle onprem vs offprem databases etc var gceProject = "shuffle" var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" var baseDockerName = "frikky/shuffle" var dbclient *datastore.Client type Userapi struct { Username string `datastore:"Username"` ApiKey string `datastore:"apikey"` } type ExecutionInfo struct { TotalApiUsage int64 `json:"total_api_usage" datastore:"total_api_usage"` TotalWorkflowExecutions int64 `json:"total_workflow_executions" datastore:"total_workflow_executions"` TotalAppExecutions int64 `json:"total_app_executions" datastore:"total_app_executions"` TotalCloudExecutions int64 `json:"total_cloud_executions" datastore:"total_cloud_executions"` TotalOnpremExecutions int64 `json:"total_onprem_executions" datastore:"total_onprem_executions"` DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` DailyAppExecutions int64 `json:"daily_app_executions" datastore:"daily_app_executions"` DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"` } type StatisticsData struct { Timestamp int64 `json:"timestamp" datastore:"timestamp"` Id string `json:"id" datastore:"id"` Amount int64 `json:"amount" datastore:"amount"` } type StatisticsItem struct { Total int64 `json:"total" datastore:"total"` Fieldname string `json:"field_name" datastore:"field_name"` Data []StatisticsData `json:"data" datastore:"data"` } // "Execution by status" // Execution history //type GlobalStatistics struct { // BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"` // WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` // ExecutionCount int64 `json:"execution_count" datastore:"execution_count"` // ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"` // ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"` // ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"` // ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"` // AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"` // TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"` // SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"` // WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"` // Baseline map[string]int64 `json:"baseline" datastore:"baseline"` //} type ParsedOpenApi struct { Body string `datastore:"body,noindex" json:"body"` ID string `datastore:"id" json:"id"` Success bool `datastore:"success,omitempty" json:"success,omitempty"` } // Limits set for a user so that they can't do a shitload type UserLimits struct { DailyApiUsage int64 `json:"daily_api_usage" datastore:"daily_api_usage"` DailyWorkflowExecutions int64 `json:"daily_workflow_executions" datastore:"daily_workflow_executions"` DailyCloudExecutions int64 `json:"daily_cloud_executions" datastore:"daily_cloud_executions"` DailyTriggers int64 `json:"daily_triggers" datastore:"daily_triggers"` DailyMailUsage int64 `json:"daily_mail_usage" datastore:"daily_mail_usage"` MaxTriggers int64 `json:"max_triggers" datastore:"max_triggers"` MaxWorkflows int64 `json:"max_workflows" datastore:"max_workflows"` } // Saves some data, not sure what to have here lol type UserAuth struct { Description string `json:"description" datastore:"description" yaml:"description"` Name string `json:"name" datastore:"name" yaml:"name"` Workflows []string `json:"workflows" datastore:"workflows"` Username string `json:"username" datastore:"username"` Fields []UserAuthField `json:"fields" datastore:"fields"` } type UserAuthField struct { Key string `json:"key" datastore:"key"` Value string `json:"value" datastore:"value"` } // Not environment, but execution environment type Environment struct { Name string `datastore:"name"` Type string `datastore:"type"` Registered bool `datastore:"registered"` } type User struct { Username string `datastore:"Username"` Password string `datastore:"password,noindex"` Session string `datastore:"session,noindex"` Verified bool `datastore:"verified,noindex"` PrivateApps []WorkflowApp `datastore:"privateapps"` Role string `datastore:"role"` VerificationToken string `datastore:"verification_token"` ApiKey string `datastore:"apikey"` ResetReference string `datastore:"reset_reference"` Executions ExecutionInfo `datastore:"executions" json:"executions"` Limits UserLimits `datastore:"limits" json:"limits"` Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` ResetTimeout int64 `datastore:"reset_timeout,noindex"` Id string `datastore:"id" json:"id"` Orgs string `datastore:"orgs" json:"orgs"` CreationTime int64 `datastore:"creation_time" json:"creation_time"` } // timeout maybe? idk type session struct { Username string `datastore:"Username,noindex"` Session string `datastore:"session,noindex"` } type loginStruct struct { Username string `json:"Username"` Password string `json:"password"` } type Contact struct { Firstname string `json:"firstname"` Lastname string `json:"lastname"` Title string `json:"title"` Companyname string `json:"companyname"` Phone string `json:"phone"` Email string `json:"email"` Message string `json:"message"` } type Translator struct { Src struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` Description string `json:"description" datastore:"description"` Required string `json:"required" datastore:"required"` Type string `json:"type" datastore:"type"` Schema struct { Type string `json:"type" datastore:"type"` } `json:"schema" datastore:"schema"` } `json:"src" datastore:"src"` Dst struct { Name string `json:"name" datastore:"name"` Value string `json:"value" datastore:"value"` Type string `json:"type" datastore:"type"` Description string `json:"description" datastore:"description"` Required string `json:"required" datastore:"required"` Schema struct { Type string `json:"type" datastore:"type"` } `json:"schema" datastore:"schema"` } `json:"dst" datastore:"dst"` } type Appconfig struct { Key string `json:"key" datastore:"key"` Value string `json:"value" datastore:"value"` } type ScheduleApp struct { Foldername string `json:"foldername" datastore:"foldername,noindex"` Name string `json:"name" datastore:"name,noindex"` Id string `json:"id" datastore:"id,noindex"` Description string `json:"description" datastore:"description,noindex"` Action string `json:"action" datastore:"action,noindex"` Config []Appconfig `json:"config,omitempty" datastore:"config,noindex"` } type AppInfo struct { SourceApp ScheduleApp `json:"sourceapp,omitempty" datastore:"sourceapp,noindex"` DestinationApp ScheduleApp `json:"destinationapp,omitempty" datastore:"destinationapp,noindex"` } // May 2020: Reused for onprem schedules - Id, Seconds, WorkflowId and argument type ScheduleOld struct { Id string `json:"id" datastore:"id"` Seconds int `json:"seconds" datastore:"seconds"` WorkflowId string `json:"workflow_id datastore:"workflow_id", ` Argument string `json:"argument" datastore:"argument"` AppInfo AppInfo `json:"appinfo" datastore:"appinfo,noindex"` Finished bool `json:"finished" finished:"id"` BaseAppLocation string `json:"base_app_location" datastore:"baseapplocation,noindex"` Translator []Translator `json:"translator,omitempty" datastore:"translator"` Org string `json:"org" datastore:"org"` CreatedBy string `json:"createdby" datastore:"createdby"` Availability string `json:"availability" datastore:"availability"` CreationTime int64 `json:"creationtime" datastore:"creationtime,noindex"` LastModificationtime int64 `json:"lastmodificationtime" datastore:"lastmodificationtime,noindex"` LastRuntime int64 `json:"lastruntime" datastore:"lastruntime,noindex"` } // Returned from /GET /schedules type Schedules struct { Schedules []ScheduleOld `json:"schedules"` Success bool `json:"success"` } type ScheduleApps struct { Apps []ApiYaml `json:"apps"` Success bool `json:"success"` } // The yaml that is uploaded type ApiYaml struct { Name string `json:"name" yaml:"name" required:"true datastore:"name"` Foldername string `json:"foldername" yaml:"foldername" required:"true datastore:"foldername"` Id string `json:"id" yaml:"id",required:"true, datastore:"id"` Description string `json:"description" datastore:"description" yaml:"description"` AppVersion string `json:"app_version" yaml:"app_version",datastore:"app_version"` ContactInfo struct { Name string `json:"name" datastore:"name" yaml:"name"` Url string `json:"url" datastore:"url" yaml:"url"` } `json:"contact_info" datastore:"contact_info" yaml:"contact_info"` Types []string `json:"types" datastore:"types" yaml:"types"` Input []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` InputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` } `json:"schema" datastore:"schema" yaml:"schema"` } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` OutputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` } `json:"schema" datastore:"schema" yaml:"schema"` } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` Config []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` } `json:"schema" datastore:"schema" yaml:"schema"` } `json:"config" datastore:"config" yaml:"config"` } `json:"input" datastore:"input" yaml:"input"` Output []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Config []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` } `json:"schema" datastore:"schema" yaml:"schema"` } `json:"config" datastore:"config" yaml:"config"` InputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` } `json:"schema" datastore:"schema" yaml:"schema"` } `json:"inputparameters" datastore:"inputparameters" yaml:"inputparameters"` OutputParameters []struct { Name string `json:"name" datastore:"name" yaml:"name"` Description string `json:"description" datastore:"description" yaml:"description"` Required string `json:"required" datastore:"required" yaml:"required"` Schema struct { Type string `json:"type" datastore:"type" yaml:"type"` } `json:"schema" datastore:"schema" yaml:"schema"` } `json:"outputparameters" datastore:"outputparameters" yaml:"outputparameters"` } `json:"output" datastore:"output" yaml:"output"` } type Hooks struct { Hooks []Hook `json:"hooks"` Success bool `json:"-"` } type Info struct { Url string `json:"url" datastore:"url"` Name string `json:"name" datastore:"name"` Description string `json:"description" datastore:"description"` } // Actions to be done by webhooks etc // Field is the actual field to use from json type HookAction struct { Type string `json:"type" datastore:"type"` Name string `json:"name" datastore:"name"` Id string `json:"id" datastore:"id"` Field string `json:"field" datastore:"field"` } type Hook struct { Id string `json:"id" datastore:"id"` Info Info `json:"info" datastore:"info"` Actions []HookAction `json:"actions" datastore:"actions"` Type string `json:"type" datastore:"type"` Owner string `json:"owner" datastore:"owner"` Status string `json:"status" datastore:"status"` Workflows []string `json:"workflows" datastore:"workflows"` Running bool `json:"running" datastore:"running"` } func createFileFromFile(ctx context.Context, bucket *storage.BucketHandle, remotePath, localPath string) error { // [START upload_file] f, err := os.Open(localPath) if err != nil { return err } defer f.Close() wc := bucket.Object(remotePath).NewWriter(ctx) if _, err = io.Copy(wc, f); err != nil { return err } if err := wc.Close(); err != nil { return err } // [END upload_file] return nil } func createFileFromBytes(ctx context.Context, bucket *storage.BucketHandle, remotePath string, data []byte) error { wc := bucket.Object(remotePath).NewWriter(ctx) byteReader := bytes.NewReader(data) if _, err := io.Copy(wc, byteReader); err != nil { return err } if err := wc.Close(); err != nil { return err } // [END upload_file] return nil } func deleteFile(ctx context.Context, bucket *storage.BucketHandle, remotePath string) error { // [START delete_file] o := bucket.Object(remotePath) if err := o.Delete(ctx); err != nil { return err } // [END delete_file] return nil } func readFile(ctx context.Context, bucket *storage.BucketHandle, object string) ([]byte, error) { // [START download_file] rc, err := bucket.Object(object).NewReader(ctx) if err != nil { return nil, err } defer rc.Close() data, err := ioutil.ReadAll(rc) if err != nil { return nil, err } return data, nil // [END download_file] } func IndexHandler(entrypoint string) func(w http.ResponseWriter, r *http.Request) { fn := func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, entrypoint) } return http.HandlerFunc(fn) } func GetUsersHandler(w http.ResponseWriter, r *http.Request) { data := map[string]interface{}{ "id": "12345", "ts": time.Now().Format(time.RFC3339), } b, err := json.Marshal(data) if err != nil { http.Error(w, err.Error(), 400) return } w.Write(b) } func jsonPrettyPrint(in string) string { var out bytes.Buffer err := json.Indent(&out, []byte(in), "", "\t") if err != nil { return in } return out.String() } // Does User exist? // Does User have permission to view / run this? // Encoding: /json? // General authentication func authenticate(request *http.Request) bool { authField := "authorization" authenticationKey := "topkek" //authFound := false // This should work right? for name, headers := range request.Header { name = strings.ToLower(name) for _, h := range headers { if name == authField && h == authenticationKey { //log.Printf("%v: %v", name, h) return true } } } return false } func publishPubsub(ctx context.Context, topic string, data []byte, attributes map[string]string) error { client, err := pubsub.NewClient(ctx, gceProject) if err != nil { return err } t := client.Topic(topic) result := t.Publish(ctx, &pubsub.Message{ Data: data, Attributes: attributes, }) // Block until the result is returned and a server-generated // ID is returned for the published message. id, err := result.Get(ctx) if err != nil { return err } log.Printf("Published message for topic %s; msg ID: %v\n", topic, id) return nil } func checkError(cmdName string, cmdArgs []string) error { cmd := exec.Command(cmdName, cmdArgs...) cmdReader, err := cmd.StdoutPipe() if err != nil { fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err) return err } scanner := bufio.NewScanner(cmdReader) go func() { for scanner.Scan() { fmt.Printf("Out: %s\n", scanner.Text()) } }() err = cmd.Start() if err != nil { fmt.Fprintln(os.Stderr, "Error starting Cmd", err) return err } err = cmd.Wait() if err != nil { fmt.Fprintln(os.Stderr, "Error waiting for Cmd", err) return err } return nil } func md5sum(data []byte) string { hasher := md5.New() hasher.Write(data) newmd5 := hex.EncodeToString(hasher.Sum(nil)) return newmd5 } func md5sumfile(filepath string) string { dat, err := ioutil.ReadFile(filepath) if err != nil { log.Printf("Error in dat: %s", err) } hasher := md5.New() hasher.Write(dat) newmd5 := hex.EncodeToString(hasher.Sum(nil)) log.Printf("%s: %s", filepath, newmd5) return newmd5 } func checkFileExistsLocal(basepath string, filepath string) bool { User := "test" // md5sum // get tmp/results/md5sum/folder/results.json // parse /tmp/results/md5sum/results.json path := fmt.Sprintf("%s/%s", basepath, md5sumfile(filepath)) if _, err := os.Stat(path); os.IsNotExist(err) { //log.Printf("File error for %s: %s", filepath, err) return false } log.Printf("File %s exists. Getting for User %s.", filepath, User) return true } func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (User, error) { apikey := request.Header.Get("Authorization") if len(apikey) > 0 { if !strings.HasPrefix(apikey, "Bearer ") { log.Printf("Apikey doesn't start with bearer") return User{}, errors.New("No bearer token for authorization header") } apikeyCheck := strings.Split(apikey, " ") if len(apikeyCheck) != 2 { log.Printf("Invalid format for apikey.") return User{}, errors.New("Invalid format for apikey") } // fml //log.Println(apikeyCheck) // This is annoying af and is done because of maxlength lol newApikey := apikeyCheck[1] if len(newApikey) > 249 { newApikey = newApikey[0:248] } ctx := context.Background() //if item, err := memcache.Get(ctx, newApikey); err == memcache.ErrCacheMiss { // // Not in cache //} else if err != nil { // // Error with cache // log.Printf("Error getting item: %v", err) //} else { // var Userdata User // err = json.Unmarshal(item.Value, &Userdata) // if err == nil { // if len(Userdata.Username) > 0 { // return Userdata, nil // } else { // return Userdata, errors.New("User is invalid") // } // } //} // Make specific check for just service user? // Get the user based on APIkey here //log.Println(apikeyCheck[1]) Userdata, err := getApikey(ctx, apikeyCheck[1]) if err != nil { log.Printf("Apikey %s doesn't exist: %s", apikey, err) return User{}, err } // Caching both bad and good apikeys :) //b, err := json.Marshal(Userdata) //if err != nil { // log.Printf("Failed marshalling: %s", err) // return User{}, err //} // Add to cache if it doesn't exist //item := &memcache.Item{ // Key: newApikey, // Value: b, // Expiration: time.Minute * 60, //} //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { // if err := memcache.Set(ctx, item); err != nil { // log.Printf("Error setting item: %v", err) // } //} else if err != nil { // log.Printf("error adding item: %v", err) //} else { // log.Printf("Set cache for %s", item.Key) //} if len(Userdata.Username) > 0 { return Userdata, nil } else { return Userdata, errors.New("User is invalid") } } // One time API keys authorizationArr, ok := request.URL.Query()["authorization"] ctx := context.Background() if ok { authorization := "" if len(authorizationArr) > 0 { authorization = authorizationArr[0] } _ = authorization //if item, err := memcache.Get(ctx, authorization); err == memcache.ErrCacheMiss { // // Doesn't exist :( // log.Printf("Couldn't find %s in cache!", authorization) // return User{}, err //} else if err != nil { // log.Printf("Error getting item: %v", err) // return User{}, err //} else { // log.Printf("%#v", item.Value) // var Userdata User // log.Printf("Deleting key %s", authorization) // memcache.Delete(ctx, authorization) // err = json.Unmarshal(item.Value, &Userdata) // if err == nil { // return Userdata, nil // } // return User{}, err //} } c, err := request.Cookie("session_token") if err == nil { //if item, err := memcache.Get(ctx, c.Value); err == memcache.ErrCacheMiss { // // Not in cache //} else if err != nil { // log.Printf("Error getting item: %v", err) //} else { // var Userdata User // err = json.Unmarshal(item.Value, &Userdata) // if err == nil { // return Userdata, nil // } //} sessionToken := c.Value session, err := getSession(ctx, sessionToken) if err != nil { log.Printf("Session %s doesn't exist (api auth): %s", sessionToken, err) return User{}, err } // Get session first // Should basically never happen Userdata, err := getUser(ctx, session.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", session.Username, err) return User{}, err } if Userdata.Session != sessionToken { return User{}, errors.New("Wrong session token") } // Means session exists, but return *Userdata, nil } // Key = apikey return User{}, errors.New("Missing authentication") } func handleGetallSchedules(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } var err error var limit = 50 // FIXME - add org search and public / private key, ok := request.URL.Query()["limit"] if ok { limit, err = strconv.Atoi(key[0]) if err != nil { limit = 50 } } // Max datastore limit if limit > 1000 { limit = 1000 } // Get URLs from a database index (mapped by orborus) ctx := context.Background() q := datastore.NewQuery("schedules").Limit(limit) var allschedules Schedules _, err = dbclient.GetAll(ctx, q, &allschedules.Schedules) if err != nil { log.Println(err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting schedules"}`))) return } newjson, err := json.Marshal(allschedules) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) return } resp.WriteHeader(200) resp.Write(newjson) } func redirect(w http.ResponseWriter, req *http.Request) { // remove/add not default ports from req.Host target := "https://" + req.Host + req.URL.Path if len(req.URL.RawQuery) > 0 { target += "?" + req.URL.RawQuery } log.Printf("redirect to: %s", target) http.Redirect(w, req, target, // see @andreiavrammsd comment: often 307 > 301 http.StatusTemporaryRedirect) } func parseLoginParameters(resp http.ResponseWriter, request *http.Request) (loginStruct, error) { body, err := ioutil.ReadAll(request.Body) if err != nil { return loginStruct{}, err } var t loginStruct err = json.Unmarshal(body, &t) if err != nil { return loginStruct{}, err } return t, nil } // Can check against HIBP etc? // Removed for localhost func checkPasswordStrength(password string) error { // Check password strength here //if len(password) < 10 { // return errors.New("Minimum password length is 10.") //} //if len(password) > 128 { // return errors.New("Maximum password length is 128.") //} //re := regexp.MustCompile("[0-9]+") //if len(re.FindAllString(password, -1)) == 0 { // return errors.New("Password must contain a number") //} //re = regexp.MustCompile("[a-z]+") //if len(re.FindAllString(password, -1)) == 0 { // return errors.New("Password must contain a lower case char") //} //re = regexp.MustCompile("[A-Z]+") //if len(re.FindAllString(password, -1)) == 0 { // return errors.New("Password must contain an upper case char") //} return nil } // No more emails :) func checkUsername(Username string) error { // Stupid first check of email loool //if !strings.Contains(Username, "@") || !strings.Contains(Username, ".") { // return errors.New("Invalid Username") //} if len(Username) < 4 { return errors.New("Minimum Username length is 4") } return nil } func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } defaultMessage := "Successfully registered" var reference string location := strings.Split(request.URL.String(), "/") if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } reference = location[4] if len(reference) != 36 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Id when registering verification is not valid"}`)) return } ctx := context.Background() // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("verification_token =", reference) var users []User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users for verification token: %s", err) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } // FIXME - check reset_timeout if len(users) != 1 { log.Printf("Error - no user with verification id %s", reference) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } Userdata := users[0] // FIXME: Not for cloud! Userdata.Verified = true err = setUser(ctx, &Userdata) if err != nil { log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) log.Printf("%s SUCCESSFULLY FINISHED REGISTRATION", Userdata.Username) } func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // FIXME: Overhaul the top part. // Only admin can change environments, but if there are no users, anyone can make (first) user, err := handleApiAuthentication(resp, request) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`)) return } if user.Role != "admin" { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`)) return } ctx := context.Background() var environments []Environment q := datastore.NewQuery("Environments") _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't get environments when setting"}`)) return } body, err := ioutil.ReadAll(request.Body) if err != nil { log.Println("Failed reading body") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) return } var newEnvironments []Environment err = json.Unmarshal(body, &newEnvironments) if err != nil { log.Printf("Failed unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) return } if len(newEnvironments) < 1 { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "One environment is required"}`))) return } // Clear old data for _, item := range environments { err = DeleteKey(ctx, "Environments", item.Name) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Error cleaning up environment"}`)) return } } for _, item := range newEnvironments { err = setEnvironment(ctx, &item) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed setting environment variable"}`)) return } } // FIXME - check which are in use log.Printf("FIXME: Set new environments: %#v", newEnvironments) log.Printf("DONT DELETE ONES THAT ARE IN USE") resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } func handleRegister(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // FIXME: Overhaul the top part. // Only admin can CREATE users, but if there are no users, anyone can make (first) count, countErr := getUserCount() user, err := handleApiAuthentication(resp, request) if err != nil { if (countErr == nil && count > 0) || countErr != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin"}`)) return } } //log.Printf("User role: %s", user.Role) if err == nil && user.Role != "admin" && count > 0 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't register without being admin (2)"}`)) return } // Gets a struct of Username, password data, err := parseLoginParameters(resp, request) if err != nil { log.Printf("Invalid params: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } // Returns false if there is an issue // Use this for register err = checkPasswordStrength(data.Password) if err != nil { log.Printf("Bad password strength: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } err = checkUsername(data.Username) if err != nil { log.Printf("Bad Username strength: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } // FIXME - use it somehow ctx := context.Background() _, err = getUser(ctx, data.Username) if err == nil { log.Printf("Username %s exists and can't register", data.Username) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } hashedPassword, err := bcrypt.GenerateFromPassword([]byte(data.Password), 8) if err != nil { log.Printf("Wrong password for %s: %s", data.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } newUser := new(User) newUser.Username = data.Username newUser.Password = string(hashedPassword) newUser.Verified = false newUser.Role = "user" newUser.CreationTime = time.Now().Unix() // FIXME - Remove this later newUser.Role = "admin" // set limits // WorkflowExecutions > CloudExecutions simply because of onprem newUser.Limits.DailyApiUsage = 100 newUser.Limits.DailyWorkflowExecutions = 1000 newUser.Limits.DailyCloudExecutions = 100 newUser.Limits.DailyTriggers = 20 newUser.Limits.DailyMailUsage = 100 newUser.Limits.MaxTriggers = 10 newUser.Limits.MaxWorkflows = 10 // Set base info for the user newUser.Executions.TotalApiUsage = 0 newUser.Executions.TotalWorkflowExecutions = 0 newUser.Executions.TotalAppExecutions = 0 newUser.Executions.TotalCloudExecutions = 0 newUser.Executions.TotalOnpremExecutions = 0 newUser.Executions.DailyApiUsage = 0 newUser.Executions.DailyWorkflowExecutions = 0 newUser.Executions.DailyAppExecutions = 0 newUser.Executions.DailyCloudExecutions = 0 newUser.Executions.DailyOnpremExecutions = 0 addr := newUser.Username verifyToken := uuid.NewV4() ID := uuid.NewV4() newUser.Id = ID.String() newUser.VerificationToken = verifyToken.String() err = setUser(ctx, newUser) if err != nil { log.Printf("Error adding User %s: %s", data.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } url := fmt.Sprintf("https://shuffler.io/register/%s", verifyToken.String()) const verifyMessage = ` Registration URL :) %s ` msg := &mail.Message{ Sender: "Shuffle ", To: []string{addr}, Subject: "Verify your username - Shuffle", Body: fmt.Sprintf(verifyMessage, url), } log.Println(msg.Body) if err := mail.Send(ctx, msg); err != nil { log.Printf("Couldn't send email: %v", err) } //sessionToken := uuid.NewV4() //// Finally, we set the client cookie for "session_token" as the session token we just generated //// we also set an expiry time of 120 seconds, the same as the cache //http.SetCookie(resp, &http.Cookie{ // Name: "session_token", // Value: sessionToken.String(), // Expires: time.Now().Add(1200 * time.Second), //}) //log.Println(Userdata) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) log.Printf("%s Successfully registered.", data.Username) //err = SetSession(*newUser, sessionToken.String()) //if err != nil { // log.Printf("Error adding session to database: %s", err) //} //err = SetApikey(*newUser) //if err != nil { // log.Printf("Error adding apikey to database: %s", err) //} //err = SetSession(*newUser, sessionToken.String()) //if err != nil { // log.Printf("Error adding apikey to database: %s", err) //} } func handleCookie(request *http.Request) bool { c, err := request.Cookie("session_token") if err != nil { return false } if len(c.Value) == 0 { return false } return true } func handleLogout(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Check cookie c, err := request.Cookie("session_token") if err != nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } else { log.Printf("Session cookie is set!") } var Userdata User ctx := context.Background() //item, err := memcache.Get(ctx, c.Value) sessionToken := "" //// Memcache handling for logout //if err == nil { // err = json.Unmarshal(item.Value, &Userdata) // if err != nil { // log.Printf("Failed unmarshaling: %s", err) // resp.WriteHeader(401) // resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) // return // } // sessionToken = Userdata.Session //} else { // // Validate with User sessionToken = c.Value session, err := getSession(ctx, sessionToken) if err != nil { log.Printf("Session %s doesn't exist (logout): %s", session.Session, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // Get session first // Should basically never happen _, err = getUser(ctx, session.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } // Userdata = *tmpdata //} // FIXME // Session might delete someone elses here? // No need to think about before possible scale..? err = SetSession(ctx, Userdata, "") if err != nil { log.Printf("Error removing session for: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } err = DeleteKey(ctx, "sessions", sessionToken) if err != nil { log.Printf("Error deleting key %s for %s: %s", c.Value, Userdata.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } Userdata.Session = "" err = setUser(ctx, &Userdata) if err != nil { log.Printf("Failed updating user: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed updating apikey"}`)) return } //memcache.Delete(request.Context(), sessionToken) resp.WriteHeader(200) resp.Write([]byte(`{"success": false, "reason": "Successfully logged out"}`)) http.SetCookie(resp, c) } func generateApikey(ctx context.Context, userInfo User) (User, error) { // Generate UUID // Set uuid to apikey in backend (update) apikey := uuid.NewV4() userInfo.ApiKey = apikey.String() err := SetApikey(ctx, userInfo) if err != nil { log.Printf("Failed updating apikey: %s", err) return userInfo, err } // Updating user err = setUser(ctx, &userInfo) if err != nil { log.Printf("Failed updating user: %s", err) return userInfo, err } return userInfo, nil } func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } c, err := request.Cookie("session_token") if err != nil { log.Printf("User doesn't have sessiontoken, on apigen: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } ctx := context.Background() sessionToken := c.Value session, err := getSession(ctx, sessionToken) if err != nil { log.Printf("Session %#v doesn't exist (api gen): %s", session, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // Get session first // Should basically never happen userInfo, err := getUser(ctx, session.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // Delete old apikey from cache //memcache.Delete(ctx, userInfo.ApiKey) if session.Session != userInfo.Session { log.Printf("Session %s is not the latest. %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } newUserInfo, err := generateApikey(ctx, *userInfo) if err != nil { log.Printf("Failed to generate apikey for user %s: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } userInfo = &newUserInfo //memcache.Delete(request.Context(), sessionToken) log.Printf("Updated apikey for user %s", userInfo.Username) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "Username": "%s", "verified": %t, "apikey": "%s"}`, userInfo.Username, userInfo.Verified, userInfo.ApiKey))) } func handleSettings(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } c, err := request.Cookie("session_token") if err != nil { log.Printf("User doesn't have sessiontoken, on getsettings: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } ctx := context.Background() sessionToken := c.Value session, err := getSession(ctx, sessionToken) if err != nil { log.Printf("Session %#v doesn't exist (settings): %s", session, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // Get session first // Should basically never happen UserInfo, err := getUser(ctx, session.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } //log.Printf("%s %s", session.Session, UserInfo.Session) if session.Session != UserInfo.Session { log.Printf("Session %s is not the latest. %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "Username": "%s", "verified": %t, "apikey": "%s"}`, UserInfo.Username, UserInfo.Verified, UserInfo.ApiKey))) } func handleInfo(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Should compare with local storage first c, err := request.Cookie("session_token") if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } // FIXME - check memcache here // Get the item from the memcache ctx := context.Background() sessionToken := c.Value session, err := getSession(ctx, sessionToken) if err != nil { //log.Printf("Session %#v doesn't exist: %s", session, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // Get session first // Should basically never happen UserInfo, err := getUser(ctx, session.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } //log.Printf("%s %s", session.Session, UserInfo.Session) if session.Session != UserInfo.Session { log.Printf("Session %s is not the latest. %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } expiration := time.Now().Add(3600 * time.Second) http.SetCookie(resp, &http.Cookie{ Name: "session_token", Value: UserInfo.Session, Expires: expiration, }) returnData := fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, UserInfo.Session, expiration.Unix()) resp.WriteHeader(200) resp.Write([]byte(returnData)) } type passwordReset struct { Password1 string `json:"newpassword"` Password2 string `json:"newpassword2"` Reference string `json:"reference"` } type passwordChange struct { Password1 string `json:"newpassword"` Password2 string `json:"newpassword2"` Password3 string `json:"currentpassword"` } func handlePasswordResetMail(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } log.Println("Handling password reset mail") defaultMessage := "We have sent you an email :)" body, err := ioutil.ReadAll(request.Body) if err != nil { log.Println("Failed reading body") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) return } type passwordReset struct { Username string `json:"Username"` } var t passwordReset err = json.Unmarshal(body, &t) if err != nil { log.Printf("Failed unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, defaultMessage))) return } ctx := context.Background() Userdata, err := getUser(ctx, t.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", t.Username, err) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) return } resetToken := uuid.NewV4() // FIXME: // Weakness with this system is that you can spam someone with password resets, // and they would never be able to reset, as a new token is always generated url := fmt.Sprintf("https://shuffler.io/passwordreset/%s", resetToken.String()) Userdata.ResetReference = resetToken.String() Userdata.ResetTimeout = 0 err = setUser(ctx, Userdata) if err != nil { log.Printf("Error patching User for mail %s: %s", Userdata.Username, err) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) return } log.Printf("%#v", Userdata) addr := t.Username const confirmMessage = ` Reset URL :) %s ` msg := &mail.Message{ Sender: "Shuffle ", To: []string{addr}, Subject: "Reset your password - Shuffle", Body: fmt.Sprintf(confirmMessage, url), } log.Println(msg.Body) if err := mail.Send(ctx, msg); err != nil { log.Printf("Couldn't send email: %v", err) } // FIXME // Generate an email to send // Generate a reset code with a reset link // Build frontend to handle reset link with "new password" etc. resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) } func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } log.Println("Handling password reset") defaultMessage := "Successfully handled password reset" body, err := ioutil.ReadAll(request.Body) if err != nil { log.Println("Failed reading body") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } var t passwordReset err = json.Unmarshal(body, &t) if err != nil { log.Println("Failed unmarshaling") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } if t.Password1 != t.Password2 { resp.WriteHeader(401) err := "Passwords don't match" resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } if len(t.Password1) < 10 || len(t.Password2) < 10 { resp.WriteHeader(401) err := "Passwords don't match - 2" resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } ctx := context.Background() // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("reset_reference =", t.Reference) var users []User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users: %s", err) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } // FIXME - check reset_timeout if len(users) != 1 { log.Printf("Error - no user with id %s", t.Reference) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } Userdata := users[0] hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Password1), 8) if err != nil { log.Printf("Wrong password for %s: %s", Userdata.Username, err) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } Userdata.Password = string(hashedPassword) Userdata.ResetTimeout = 0 Userdata.ResetReference = "" err = setUser(ctx, &Userdata) if err != nil { log.Printf("Error adding User %s: %s", Userdata.Username, err) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) return } // FIXME - maybe send a mail here to say that the password was changed resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "%s"}`, defaultMessage))) } func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { log.Println("Handling password change") cors := handleCors(resp, request) if cors { return } body, err := ioutil.ReadAll(request.Body) if err != nil { log.Println("Failed reading body") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } var t passwordChange err = json.Unmarshal(body, &t) if err != nil { log.Println("Failed unmarshaling") resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } if t.Password1 != t.Password2 { resp.WriteHeader(401) err := "Passwords don't match" resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } if len(t.Password1) < 10 || len(t.Password2) < 10 { resp.WriteHeader(401) err := "Passwords don't match - 2" resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } err = checkPasswordStrength(t.Password3) if err != nil { log.Printf("Bad password strength: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } // Check cookie c, err := request.Cookie("session_token") if err != nil { log.Printf("User doesn't have sessiontoken on pw change: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "You're not logged in."}`))) return } ctx := context.Background() // Validate with User sessionToken := c.Value session, err := getSession(ctx, sessionToken) if err != nil { log.Printf("Session %s doesn't exist (password change): %s", session.Session, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "You're not logged in"}`)) return } // Get session first // Should basically never happen Userdata, err := getUser(ctx, session.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } err = bcrypt.CompareHashAndPassword([]byte(Userdata.Password), []byte(t.Password1)) if err != nil { log.Printf("Bad password for %s: %s", session.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } hashedPassword, err := bcrypt.GenerateFromPassword([]byte(t.Password3), 8) if err != nil { log.Printf("Wrong password for %s: %s", Userdata.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } Userdata.Password = string(hashedPassword) err = setUser(ctx, Userdata) if err != nil { log.Printf("Error adding User %s: %s", Userdata.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } //memcache.Delete(ctx, sessionToken) resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } body, err := ioutil.ReadAll(request.Body) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } var t Contact err = json.Unmarshal(body, &t) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } if len(t.Email) < 3 || len(t.Message) == 0 { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Please fill a valid email and message"}`))) return } ctx := context.Background() mailContent := fmt.Sprintf("Firsname: %s\nLastname: %s\nTitle: %s\nCompanyname: %s\nPhone: %s\nEmail: %s\nMessage: %s", t.Firstname, t.Lastname, t.Title, t.Companyname, t.Phone, t.Email, t.Message) log.Printf("Sending contact from %s", t.Email) msg := &mail.Message{ Sender: "Shuffle ", To: []string{"frikky@shuffler.io"}, Subject: "Shuffler.io - New contact form", Body: mailContent, } if err := mail.Send(ctx, msg); err != nil { log.Printf("Couldn't send email: %v", err) } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`))) } func getEnvironmentCount() (int, error) { ctx := context.Background() q := datastore.NewQuery("Environments").Limit(1) count, err := dbclient.Count(ctx, q) if err != nil { return 0, err } return count, nil } func getUserCount() (int, error) { ctx := context.Background() q := datastore.NewQuery("Users").Limit(1) count, err := dbclient.Count(ctx, q) if err != nil { return 0, err } return count, nil } func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } _, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } ctx := context.Background() var environments []Environment q := datastore.NewQuery("Environments") _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't get environments"}`)) return } newjson, err := json.Marshal(environments) if err != nil { log.Printf("Failed unmarshal: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking environments"}`))) return } //log.Printf("Existing environments: %s", string(newjson)) resp.WriteHeader(200) resp.Write(newjson) } func handleGetUsers(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if user.Role != "admin" { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Not admin"}`)) return } ctx := context.Background() var users []User q := datastore.NewQuery("Users") _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't get users"}`)) return } newUsers := []User{} for _, item := range users { if len(item.Username) == 0 { continue } item.Password = "" item.Session = "" item.VerificationToken = "" newUsers = append(newUsers, item) } newjson, err := json.Marshal(newUsers) if err != nil { log.Printf("Failed unmarshal: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) return } resp.WriteHeader(200) resp.Write(newjson) } func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } count, err := getUserCount() if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) } if count == 0 { log.Printf("No users - redirecting for management user") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "stay"}`))) return } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect"}`))) } func handleLogin(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Gets a struct of Username, password data, err := parseLoginParameters(resp, request) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } log.Printf("Handling login of %s", data.Username) err = checkUsername(data.Username) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } ctx := context.Background() Userdata, err := getUser(ctx, data.Username) if err != nil { log.Printf("Username %s doesn't exist: %s", data.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } err = bcrypt.CompareHashAndPassword([]byte(Userdata.Password), []byte(data.Password)) if err != nil { log.Printf("Password for %s is incorrect: %s", data.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Username and/or password is incorrect"}`)) return } log.Printf("%s SUCCESSFULLY LOGGED IN", data.Username) //if !Userdata.Verified { // log.Printf("User %s is not verified", data.Username) // resp.WriteHeader(403) // resp.Write([]byte(`{"success": false, "reason": "Successful login, but your email address isn't verified. Check your mailbox."}`)) // return //} loginData := `{"success": true}` // FIXME - have timeout here if len(Userdata.Session) != 0 { //log.Println("Nonexisting session") expiration := time.Now().Add(3600 * time.Second) http.SetCookie(resp, &http.Cookie{ Name: "session_token", Value: Userdata.Session, Expires: expiration, }) loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix()) //log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session) err = SetSession(ctx, *Userdata, Userdata.Session) if err != nil { log.Printf("Error adding session to database: %s", err) } resp.WriteHeader(200) resp.Write([]byte(loginData)) return } sessionToken := uuid.NewV4() http.SetCookie(resp, &http.Cookie{ Name: "session_token", Value: sessionToken.String(), Expires: time.Now().Add(3600 * time.Second), }) // ADD TO DATABASE err = SetSession(ctx, *Userdata, sessionToken.String()) if err != nil { log.Printf("Error adding session to database: %s", err) } resp.WriteHeader(200) resp.Write([]byte(loginData)) } func getApikey(ctx context.Context, apikey string) (User, error) { // Query for the specifci workflowId q := datastore.NewQuery("Users").Filter("apikey =", apikey) var users []User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Error getting users apikey: %s", err) return User{}, err } if len(users) == 0 { log.Printf("No users found for apikey %s", apikey) return User{}, err } return users[0], nil } func getSession(ctx context.Context, thissession string) (*session, error) { key := datastore.NameKey("sessions", thissession, nil) curUser := &session{} if err := dbclient.Get(ctx, key, curUser); err != nil { return &session{}, err } return curUser, nil } // ListBooks returns a list of books, ordered by title. func getUser(ctx context.Context, Username string) (*User, error) { key := datastore.NameKey("Users", strings.ToLower(Username), nil) curUser := &User{} if err := dbclient.Get(ctx, key, curUser); err != nil { return &User{}, err } return curUser, nil } // Index = Username func DeleteKey(ctx context.Context, entity string, value string) error { // Non indexed User data key1 := datastore.NameKey(entity, value, nil) err := dbclient.Delete(ctx, key1) if err != nil { log.Printf("Error deleting %s from %s: %s", value, entity, err) return err } return nil } // Index = Username func SetApikey(ctx context.Context, Userdata User) error { // Non indexed User data newapiUser := new(Userapi) newapiUser.ApiKey = Userdata.ApiKey newapiUser.Username = Userdata.Username key1 := datastore.NameKey("apikey", newapiUser.ApiKey, nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key1, newapiUser); err != nil { log.Printf("Error adding apikey: %s", err) return err } return nil } // Index = Username func SetSession(ctx context.Context, Userdata User, value string) error { // Non indexed User data Userdata.Session = value key1 := datastore.NameKey("Users", strings.ToLower(Userdata.Username), nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key1, &Userdata); err != nil { log.Printf("rror adding Usersession: %s", err) return err } if len(Userdata.Session) > 0 { // Indexed session data sessiondata := new(session) sessiondata.Username = Userdata.Username sessiondata.Session = Userdata.Session key2 := datastore.NameKey("sessions", sessiondata.Session, nil) if _, err := dbclient.Put(ctx, key2, sessiondata); err != nil { log.Printf("Error adding session: %s", err) return err } } return nil } func setOpenApiDatastore(ctx context.Context, id string, data ParsedOpenApi) error { k := datastore.NameKey("openapi3", id, nil) if _, err := dbclient.Put(ctx, k, &data); err != nil { log.Println(err) return err } return nil } func getOpenApiDatastore(ctx context.Context, id string) (ParsedOpenApi, error) { key := datastore.NameKey("openapi3", id, nil) api := &ParsedOpenApi{} if err := dbclient.Get(ctx, key, api); err != nil { return ParsedOpenApi{}, err } return *api, nil } func setEnvironment(ctx context.Context, data *Environment) error { // clear session_token and API_token for user k := datastore.NameKey("Environments", strings.ToLower(data.Name), nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, k, data); err != nil { log.Println(err) return err } return nil } // ListBooks returns a list of books, ordered by title. func setUser(ctx context.Context, data *User) error { // clear session_token and API_token for user k := datastore.NameKey("Users", strings.ToLower(data.Username), nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, k, data); err != nil { log.Println(err) return err } return nil } func handleCors(resp http.ResponseWriter, request *http.Request) bool { // FIXME - this is to handle multiple frontends in test rofl origin := request.Header["Origin"] resp.Header().Set("Vary", "Origin") if len(origin) > 0 { resp.Header().Set("Access-Control-Allow-Origin", origin[0]) } else { resp.Header().Set("Access-Control-Allow-Origin", "http://localhost:4201") } //resp.Header().Set("Access-Control-Allow-Origin", "http://localhost:8000") resp.Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, X-Requested-With, remember-me") resp.Header().Set("Access-Control-Allow-Methods", "POST, GET, PUT, DELETE") resp.Header().Set("Access-Control-Allow-Credentials", "true") if request.Method == "OPTIONS" { resp.WriteHeader(200) resp.Write([]byte("OK")) return true } return false } func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (map[string]interface{}, error) { body, err := ioutil.ReadAll(request.Body) if err != nil { return nil, err } log.Printf("Parsing data: %s", string(body)) var t map[string]interface{} err = json.Unmarshal(body, &t) if err == nil { log.Printf("PARSED!! :)") return t, nil } // Translate XML to json in case of an XML blob. // FIXME - use Content-Type and Accept headers xml := strings.NewReader(string(body)) curjson, err := xj.Convert(xml) if err != nil { return t, err } //fmt.Println(curjson.String()) //log.Printf("Parsing json a second time: %s", string(curjson.String())) err = json.Unmarshal(curjson.Bytes(), &t) if err != nil { return t, nil } envelope := t["Envelope"].(map[string]interface{}) curbody := envelope["Body"].(map[string]interface{}) //log.Println(curbody) // ALWAYS handle strings only // FIXME - remove this and get it from config or something requiredField := "symptomDescription" _, found := SearchNested(curbody, requiredField) // Maxdepth maxiter := 5 // Need to look for parent of the item, as that is most likely root if found { cnt := 0 var previousDifferentItem map[string]interface{} var previousItem map[string]interface{} _ = previousItem for { if cnt == maxiter { break } // Already know it exists key, realItem, _ := SearchNestedParent(curbody, requiredField) // First should ALWAYS work since we already have recursion checked if len(previousDifferentItem) == 0 { previousDifferentItem = realItem.(map[string]interface{}) } switch t := realItem.(type) { case map[string]interface{}: previousItem = realItem.(map[string]interface{}) curbody = realItem.(map[string]interface{}) default: // Gets here if it's not an object _ = t //log.Printf("hi %#v", previousItem) return previousItem, nil } _ = key cnt += 1 } } //key, realItem, found = SearchNestedParent(newbody, requiredField) //if !found { // log.Println("NOT FOUND!") //} ////log.Println(realItem[requiredField].(map[string]interface{})) //log.Println(realItem[requiredField]) //log.Printf("FOUND PARENT :): %s", key) return t, nil } // SearchNested searches a nested structure consisting of map[string]interface{} // and []interface{} looking for a map with a specific key name. // If found SearchNested returns the value associated with that key, true func SearchNestedParent(obj interface{}, key string) (string, interface{}, bool) { switch t := obj.(type) { case map[string]interface{}: if v, ok := t[key]; ok { return "", v, ok } for k, v := range t { if _, ok := SearchNested(v, key); ok { return k, v, ok } } case []interface{}: for _, v := range t { if _, ok := SearchNested(v, key); ok { return "", v, ok } } } return "", nil, false } // SearchNested searches a nested structure consisting of map[string]interface{} // and []interface{} looking for a map with a specific key name. // If found SearchNested returns the value associated with that key, true // If the key is not found SearchNested returns nil, false func SearchNested(obj interface{}, key string) (interface{}, bool) { switch t := obj.(type) { case map[string]interface{}: if v, ok := t[key]; ok { return v, ok } for _, v := range t { if result, ok := SearchNested(v, key); ok { return result, ok } } case []interface{}: for _, v := range t { if result, ok := SearchNested(v, key); ok { return result, ok } } } return nil, false } func handleSetHook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } // FIXME - check basic authentication body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Error with body read: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Println(jsonPrettyPrint(string(body))) var hook Hook err = json.Unmarshal(body, &hook) if err != nil { log.Printf("Failed unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if user.Id != hook.Owner && user.Role != "admin" && user.Role != "scheduler" { log.Printf("Wrong user (%s) for hook %s", user.Username, hook.Id) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if hook.Id != workflowId { errorstring := fmt.Sprintf(`Id %s != %s`, hook.Id, workflowId) log.Printf("Ids not matching: %s", errorstring) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, errorstring))) return } // Verifies the hook JSON. Bad verification :^) finished, errorstring := verifyHook(hook) if !finished { log.Printf("Error with hook: %s", errorstring) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "%s"}`, errorstring))) return } // Get the ID to see whether it exists // FIXME - use return and set READONLY fields (don't allow change from User) ctx := context.Background() _, err = getHook(ctx, workflowId) if err != nil { log.Printf("Failed getting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Invalid ID"}`)) return } // Update the fields err = setHook(ctx, hook) if err != nil { log.Printf("Failed setting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } // FIXME - some fields (e.g. status) shouldn't be writeable.. Meh func verifyHook(hook Hook) (bool, string) { // required fields: Id, info.name, type, status, running if hook.Id == "" { return false, "Missing required field id" } if hook.Info.Name == "" { return false, "Missing required field info.name" } // Validate type stuff validTypes := []string{"webhook"} found := false for _, key := range validTypes { if hook.Type == key { found = true break } } if !found { return false, fmt.Sprintf("Field type is invalid. Allowed: %s", strings.Join(validTypes, ", ")) } // WEbhook specific if hook.Type == "webhook" { if hook.Info.Url == "" { return false, "Missing required field info.url" } } if hook.Status == "" { return false, "Missing required field status" } validStatusFields := []string{"running", "stopped", "uninitialized"} found = false for _, key := range validStatusFields { if hook.Status == key { found = true break } } if !found { return false, fmt.Sprintf("Field status is invalid. Allowed: %s", strings.Join(validStatusFields, ", ")) } // Verify actions if len(hook.Actions) > 0 { existingIds := []string{} for index, action := range hook.Actions { if action.Type == "" { return false, fmt.Sprintf("Missing required field actions.type at index %d", index) } if action.Name == "" { return false, fmt.Sprintf("Missing required field actions.name at index %d", index) } if action.Id == "" { return false, fmt.Sprintf("Missing required field actions.id at index %d", index) } // Check for duplicate IDs for _, actionId := range existingIds { if action.Id == actionId { return false, fmt.Sprintf("actions.id %s at index %d already exists", actionId, index) } } existingIds = append(existingIds, action.Id) } } return true, "All items set" //log.Printf("%#v", hook) //Id string `json:"id" datastore:"id"` //Info Info `json:"info" datastore:"info"` //Transforms struct{} `json:"transforms" datastore:"transforms"` //Actions []HookAction `json:"actions" datastore:"actions"` //Type string `json:"type" datastore:"type"` //Status string `json:"status" datastore:"status"` //Running bool `json:"running" datastore:"running"` } func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } // FIXME - check basic authentication body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Error with body read: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } jsonPrettyPrint(string(body)) var schedule ScheduleOld err = json.Unmarshal(body, &schedule) if err != nil { log.Printf("Failed unmarshaling: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - check access etc ctx := context.Background() err = setSchedule(ctx, schedule) if err != nil { log.Printf("Failed setting schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - get some real data? resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) return } func getSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) curUser := &ScheduleOld{} if err := dbclient.Get(ctx, key, curUser); err != nil { return &ScheduleOld{}, err } return curUser, nil } func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } ctx := context.Background() schedule, err := getSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } //log.Printf("%#v", schedule.Translator[0]) b, err := json.Marshal(schedule) if err != nil { log.Printf("Failed marshalling: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - get some real data? resp.WriteHeader(200) resp.Write([]byte(b)) return } // Starts a new webhook func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } ctx := context.Background() err := DeleteKey(ctx, "schedules", workflowId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Can't delete"}`)) return } // FIXME - remove schedule too resp.WriteHeader(200) resp.Write([]byte(`{"success": true, "message": "Deleted webhook"}`)) } // Starts a new webhook func handleNewSchedule(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } randomValue := uuid.NewV4() h := md5.New() io.WriteString(h, randomValue.String()) newId := strings.ToLower(fmt.Sprintf("%X", h.Sum(nil))) // FIXME - timestamp! // FIXME - applocation - cloud function? timeNow := int64(time.Now().Unix()) schedule := ScheduleOld{ Id: newId, AppInfo: AppInfo{}, BaseAppLocation: "/home/frikky/git/shaffuru/tmp/apps", CreationTime: timeNow, LastModificationtime: timeNow, LastRuntime: timeNow, } ctx := context.Background() err := setSchedule(ctx, schedule) if err != nil { log.Printf("Failed setting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Println("Generating new schedule") resp.WriteHeader(200) resp.Write([]byte(`{"success": true, "message": "Created new service"}`)) } // Does the webhook func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // 1. Get callback data // 2. Load the configuration // 3. Execute the workflow path := strings.Split(request.URL.String(), "/") if len(path) < 4 { resp.WriteHeader(403) resp.Write([]byte(`{"success": false}`)) return } // 1. Get config with hookId //fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId) ctx := context.Background() location := strings.Split(request.URL.String(), "/") var hookId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } hookId = location[4] } // ID: webhook_ if len(hookId) != 44 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } hookId = hookId[8:len(hookId)] log.Printf("HookID: %s", hookId) hook, err := getHook(ctx, hookId) if err != nil { log.Printf("Failed getting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Printf("HOOK FOUND: %#v", hook) // Execute the workflow //executeWorkflow(resp, request) //resp.WriteHeader(200) //resp.Write([]byte(`{"success": true}`)) if hook.Status == "stopped" { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Click start to start it"}`))) return } if len(hook.Workflows) == 0 { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No workflows are defined"}`))) return } for _, item := range hook.Workflows { log.Printf("Running for workflow: %s", item) workflow := Workflow{ ID: "", } workflowExecution, executionResp, err := handleExecution(item, workflow, request) if err == nil { err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) return } resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) } } // Starts a new webhook func handleNewHook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } type requestData struct { Type string `json:"type"` Description string `json:"description"` Id string `json:"id"` Name string `json:"name"` Workflow string `json:"workflow"` } body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Body data error: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Println("Data: %s", string(body)) ctx := context.Background() var requestdata requestData err = yaml.Unmarshal([]byte(body), &requestdata) if err != nil { log.Printf("Failed unmarshaling inputdata: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Printf("%#v", requestdata) // CBA making a real thing. Already had some code lol newId := requestdata.Id if len(newId) != 36 { log.Printf("Bad ID") resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Invalid ID"}`)) return } if requestdata.Id == "" || requestdata.Name == "" { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Requires fields id and name can't be empty"}`)) return } validTypes := []string{ "webhook", } isTypeValid := false for _, thistype := range validTypes { if requestdata.Type == thistype { isTypeValid = true break } } if !(isTypeValid) { log.Printf("Type %s is not valid. Try any of these: %s", requestdata.Type, strings.Join(validTypes, ", ")) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } hook := Hook{ Id: newId, Workflows: []string{requestdata.Workflow}, Info: Info{ Name: requestdata.Name, Description: requestdata.Description, Url: fmt.Sprintf("https://shuffler.io/functions/webhooks/webhook_%s", newId), }, Type: "webhook", Owner: user.Username, Status: "uninitialized", Actions: []HookAction{ HookAction{ Type: "workflow", Name: requestdata.Name, Id: requestdata.Workflow, Field: "", }, }, Running: false, } log.Printf("Hello") // FIXME: Add cloud function execution? //b, err := json.Marshal(hook) //if err != nil { // log.Printf("Failed marshalling: %s", err) // resp.WriteHeader(401) // resp.Write([]byte(`{"success": false}`)) // return //} //environmentVariables := map[string]string{ // "FUNCTION_APIKEY": user.ApiKey, // "CALLBACKURL": "https://shuffler.io", // "HOOKID": hook.Id, //} //applocation := fmt.Sprintf("gs://%s/triggers/webhook.zip", bucketName) //hookname := fmt.Sprintf("webhook_%s", hook.Id) //err = deployWebhookFunction(ctx, hookname, defaultLocation, applocation, environmentVariables) //if err != nil { // log.Printf("Error deploying hook: %s", err) // resp.WriteHeader(401) // resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Issue with starting hook. Please wait a second and try again"}`))) // return //} hook.Status = "running" hook.Running = true err = setHook(ctx, hook) if err != nil { log.Printf("Failed setting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } log.Println("Generating new hook") resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } func sendHookResult(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } _ = user location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } ctx := context.Background() hook, err := getHook(ctx, workflowId) if err != nil { log.Printf("Failed getting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Body data error: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Printf("SET the hook results for %s to %s", workflowId, body) // FIXME - set the hook result in the DB somehow as interface{} // FIXME - should the hook do the transform? Hmm b, err := json.Marshal(hook) if err != nil { log.Printf("Failed marshalling: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } resp.WriteHeader(200) resp.Write([]byte(b)) return } func handleGetHook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 36 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } ctx := context.Background() hook, err := getHook(ctx, workflowId) if err != nil { log.Printf("Failed getting hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if user.Id != hook.Owner && user.Role != "admin" && user.Role != "scheduler" { log.Printf("Wrong user (%s) for hook %s", user.Username, hook.Id) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } b, err := json.Marshal(hook) if err != nil { log.Printf("Failed marshalling: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - get some real data? resp.WriteHeader(200) resp.Write([]byte(b)) return } func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { if request.Method != "GET" { setSpecificSchedule(resp, request) return } cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } ctx := context.Background() schedule, err := getSchedule(ctx, workflowId) if err != nil { log.Printf("Failed getting schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } //log.Printf("%#v", schedule.Translator[0]) b, err := json.Marshal(schedule) if err != nil { log.Printf("Failed marshalling: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } resp.WriteHeader(200) resp.Write([]byte(b)) } func loadYaml(fileLocation string) (ApiYaml, error) { apiYaml := ApiYaml{} yamlFile, err := ioutil.ReadFile(fileLocation) if err != nil { log.Printf("yamlFile.Get err: %s", err) return ApiYaml{}, err } err = yaml.Unmarshal([]byte(yamlFile), &apiYaml) if err != nil { return ApiYaml{}, err } return apiYaml, nil } // This should ALWAYS come from an OUTPUT func executeSchedule(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } ctx := context.Background() log.Printf("EXECUTING %s!", workflowId) idConfig, err := getSchedule(ctx, workflowId) if err != nil { log.Printf("Error getting schedule: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } // Basically the src app inputStrings := map[string]string{} for _, item := range idConfig.Translator { if item.Dst.Required == "false" { log.Println("Skipping not required") continue } if item.Src.Name == "" { errorMsg := fmt.Sprintf("Required field %s has no source", item.Dst.Name) log.Println(errorMsg) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, errorMsg))) return } inputStrings[item.Dst.Name] = item.Src.Name } configmap := map[string]string{} for _, config := range idConfig.AppInfo.SourceApp.Config { configmap[config.Key] = config.Value } // FIXME - this wont work for everything lmao functionName := strings.ToLower(idConfig.AppInfo.SourceApp.Action) functionName = strings.Replace(functionName, " ", "_", 10) cmdArgs := []string{ fmt.Sprintf("%s/%s/app.py", baseAppPath, "thehive"), fmt.Sprintf("--referenceid=%s", workflowId), fmt.Sprintf("--function=%s", functionName), } for key, value := range configmap { cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", key, value)) } // FIXME - processname baseProcess := "python3" log.Printf("Executing: %s %s", baseProcess, strings.Join(cmdArgs, " ")) execSubprocess(baseProcess, cmdArgs) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } func execSubprocess(cmdName string, cmdArgs []string) error { cmd := exec.Command(cmdName, cmdArgs...) cmdReader, err := cmd.StdoutPipe() if err != nil { fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for Cmd", err) return err } scanner := bufio.NewScanner(cmdReader) go func() { for scanner.Scan() { fmt.Printf("Out: %s\n", scanner.Text()) } }() err = cmd.Start() if err != nil { fmt.Fprintln(os.Stderr, "Error starting Cmd", err) return err } err = cmd.Wait() if err != nil { fmt.Fprintln(os.Stderr, "Error waiting for Cmd", err) return err } return nil } // This should ALWAYS come from an OUTPUT func uploadWorkflowResult(resp http.ResponseWriter, request *http.Request) { // Post to a key with random data? location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if len(workflowId) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "ID not valid"}`)) return } // FIXME - check if permission AND whether it exists // FIXME - validate ID as well ctx := context.Background() schedule, err := getSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule %s: %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // Should use generic interfaces and parse fields OR // build temporary struct based on api.yaml of the app data, err := parseWorkflowParameters(resp, request) if err != nil { log.Printf("Invalid params: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } // Get the actual fields foldername := schedule.AppInfo.SourceApp.Foldername curOutputType := schedule.AppInfo.SourceApp.Name curOutputAppOutput := schedule.AppInfo.SourceApp.Action curInputType := schedule.AppInfo.DestinationApp.Name translatormap := schedule.Translator if len(curOutputType) <= 0 { log.Printf("Id %s is invalid. Missing sourceapp name", workflowId) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } if len(foldername) == 0 { foldername = strings.ToLower(curOutputType) } if len(curOutputAppOutput) <= 0 { log.Printf("Id %s is invalid. Missing source output ", workflowId) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } if len(curInputType) <= 0 { log.Printf("Id %s is invalid. Missing destination name", workflowId) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) return } // Needs to be used for parsing properly // Might be dumb to have the yaml as a file too yamlpath := fmt.Sprintf("%s/%s/api.yaml", baseAppPath, foldername) curyaml, err := loadYaml(yamlpath) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } //validFields := []string{} requiredFields := []string{} optionalFields := []string{} for _, output := range curyaml.Output { if output.Name != curOutputAppOutput { continue } for _, outputparam := range output.OutputParameters { if outputparam.Required == "true" { if outputparam.Schema.Type == "string" { requiredFields = append(requiredFields, outputparam.Name) } else { log.Printf("Outputparam schematype %s is not implemented.", outputparam.Schema.Type) } } else { optionalFields = append(optionalFields, outputparam.Name) } } // Wont reach here unless it's the right one break } // Checks whether ALL required fields are filled for _, fieldname := range requiredFields { if data[fieldname] == nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Field %s is required"}`, fieldname))) return } else { log.Printf("%s: %s", fieldname, data[fieldname]) } } // FIXME // Verify whether it can be sent from the source to destination here // Save to DB or send it straight? Idk // Use e.g. google pubsub if cloud and maybe kafka locally // FIXME - add more types :) sourcedatamap := map[string]string{} for key, value := range data { switch v := value.(type) { case string: sourcedatamap[key] = value.(string) default: log.Printf("unexpected type %T", v) } } log.Println(data) log.Println(requiredFields) log.Println(translatormap) log.Println(sourcedatamap) outputmap := map[string]string{} for _, translator := range translatormap { if translator.Src.Type == "static" { log.Printf("%s = %s", translator.Dst.Name, translator.Src.Value) outputmap[translator.Dst.Name] = translator.Src.Value } else { log.Printf("%s = %s", translator.Dst.Name, translator.Src.Name) outputmap[translator.Dst.Name] = sourcedatamap[translator.Src.Name] } } configmap := map[string]string{} for _, config := range schedule.AppInfo.DestinationApp.Config { configmap[config.Key] = config.Value } // FIXME - add function to run // FIXME - add reference somehow // FIXME - add apikey somehow // Just package and run really? // FIXME - generate from sourceapp outputmap["function"] = "create_alert" cmdArgs := []string{ fmt.Sprintf("%s/%s/app.py", baseAppPath, foldername), } for key, value := range outputmap { cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", key, value)) } // COnfig map! for key, value := range configmap { cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", key, value)) } outputmap["referenceid"] = workflowId baseProcess := "python3" log.Printf("Executing: %s %s", baseProcess, strings.Join(cmdArgs, " ")) execSubprocess(baseProcess, cmdArgs) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } // Index = Username func setSchedule(ctx context.Context, schedule ScheduleOld) error { key1 := datastore.NameKey("schedules", strings.ToLower(schedule.Id), nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key1, &schedule); err != nil { log.Printf("Error adding schedule: %s", err) return err } return nil } //dst: {name: "title", required: "true", type: "string"} // //"title": "symptomDescription", //"description": "detailedDescription", //"type": "ticketType", //"sourceRef": "ticketId" //"name": "secureworks", //"id": "e07910a06a086c83ba41827aa00b26ed", //"description": "I AM SECUREWORKS DESC", //"action": "Get Tickets", //"config": {} //"name": "thehive", // "id": "e07910a06a086c83ba41827aa00b26ef", // "description": "I AM thehive DESC", // "action": "Add ticket", // "config": [{ // "key": "http://localhost:9000", // "value": "kZJmmn05j8wndOGDGvKg/D9eKub1itwO" // }] func getAllScheduleApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } var err error var limit = 50 // FIXME - add org search and public / private key, ok := request.URL.Query()["limit"] if ok { limit, err = strconv.Atoi(key[0]) if err != nil { limit = 50 } } // Max datastore limit if limit > 1000 { limit = 1000 } // Get URLs from a database index (mapped by orborus) ctx := context.Background() q := datastore.NewQuery("appschedules").Limit(limit) var allappschedules ScheduleApps ret, err := dbclient.GetAll(ctx, q, &allappschedules.Apps) _ = ret if err != nil { log.Printf("Failed getting all apps: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed getting apps"}`))) return } newjson, err := json.Marshal(allappschedules) if err != nil { log.Printf("Failed unmarshal: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) return } resp.WriteHeader(200) resp.Write(newjson) } func setScheduleApp(ctx context.Context, app ApiYaml, id string) error { // id = md5(appname:appversion) key1 := datastore.NameKey("appschedules", id, nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key1, &app); err != nil { log.Printf("Error adding schedule app: %s", err) return err } return nil } func findValidScheduleAppFolders(rootAppFolder string) ([]string, error) { rootFiles, err := ioutil.ReadDir(rootAppFolder) if err != nil { return []string{}, err } invalidRootFiles := []string{} invalidRootFolders := []string{} invalidAppFolders := []string{} validAppFolders := []string{} // This is dumb allowedLanguages := []string{"py", "go"} for _, rootfile := range rootFiles { if !rootfile.IsDir() { invalidRootFiles = append(invalidRootFiles, rootfile.Name()) continue } appFolderLocation := fmt.Sprintf("%s/%s", rootAppFolder, rootfile.Name()) appFiles, err := ioutil.ReadDir(appFolderLocation) if err != nil { // Invalid app folder (deleted within a few MS lol) log.Printf("%s", err) invalidRootFolders = append(invalidRootFolders, rootfile.Name()) continue } yamlFileDone := false appFileExists := false for _, appfile := range appFiles { if appfile.Name() == "api.yaml" { err := validateAppYaml( fmt.Sprintf("%s/%s", appFolderLocation, appfile.Name()), ) if err != nil { log.Printf("Error in %s: %s", fmt.Sprintf("%s/%s", rootfile.Name(), appfile.Name()), err) break } log.Printf("YAML FOR %s: %s IS VALID!!", rootfile.Name(), appfile.Name()) yamlFileDone = true } for _, language := range allowedLanguages { if appfile.Name() == fmt.Sprintf("app.%s", language) { log.Printf("Appfile found for %s", rootfile.Name()) appFileExists = true break } } } if !yamlFileDone || !appFileExists { invalidAppFolders = append(invalidAppFolders, rootfile.Name()) } else { validAppFolders = append(validAppFolders, rootfile.Name()) } } log.Printf("Invalid rootfiles: %s", strings.Join(invalidRootFiles, ", ")) log.Printf("Invalid rootfolders: %s", strings.Join(invalidRootFolders, ", ")) log.Printf("Invalid appfolders: %s", strings.Join(invalidAppFolders, ", ")) log.Printf("\n=== VALID appfolders ===\n* %s", strings.Join(validAppFolders, "\n")) return validAppFolders, err } func validateInputOutputYaml(appType string, apiYaml ApiYaml) error { if appType == "input" { for index, input := range apiYaml.Input { if input.Name == "" { return errors.New(fmt.Sprintf("YAML field name doesn't exist in index %d of Input", index)) } if input.Description == "" { return errors.New(fmt.Sprintf("YAML field description doesn't exist in index %d of Input", index)) } for paramindex, param := range input.InputParameters { if param.Name == "" { return errors.New(fmt.Sprintf("YAML field name doesn't exist in Input %s with index %d", input.Name, paramindex)) } if param.Description == "" { return errors.New(fmt.Sprintf("YAML field description doesn't exist in Input %s with index %d", input.Name, index)) } if param.Schema.Type == "" { return errors.New(fmt.Sprintf("YAML field schema.type doesn't exist in Input %s with index %d", input.Name, index)) } } } } return nil } func validateAppYaml(fileLocation string) error { /* Requires: name, description, app_version, contact_info (name), types */ apiYaml, err := loadYaml(fileLocation) if err != nil { return err } // Validate fields if apiYaml.Name == "" { return errors.New("YAML field name doesn't exist") } if apiYaml.Description == "" { return errors.New("YAML field description doesn't exist") } if apiYaml.AppVersion == "" { return errors.New("YAML field app_version doesn't exist") } if apiYaml.ContactInfo.Name == "" { return errors.New("YAML field contact_info.name doesn't exist") } if len(apiYaml.Types) == 0 { return errors.New("YAML field types doesn't exist") } // Validate types (input/ouput) validTypes := []string{"input", "output"} for _, appType := range apiYaml.Types { // Validate in here lul for _, validType := range validTypes { if appType == validType { err = validateInputOutputYaml(appType, apiYaml) if err != nil { return err } break } } } return nil } func getHook(ctx context.Context, hookId string) (*Hook, error) { key := datastore.NameKey("hooks", strings.ToLower(hookId), nil) hook := &Hook{} if err := dbclient.Get(ctx, key, hook); err != nil { return &Hook{}, err } return hook, nil } func setHook(ctx context.Context, hook Hook) error { key1 := datastore.NameKey("hooks", strings.ToLower(hook.Id), nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key1, &hook); err != nil { log.Printf("Error adding hook: %s", err) return err } return nil } func handleGetallHooks(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } ctx := context.Background() // With user, do a search for workflows with user or user's org attached q := datastore.NewQuery("hooks").Filter("owner =", user.Username) var allhooks []Hook _, err = dbclient.GetAll(ctx, q, &allhooks) if err != nil { log.Printf("Failed getting workflows for user %s: %s", user.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if len(allhooks) == 0 { resp.WriteHeader(200) resp.Write([]byte("[]")) return } newjson, err := json.Marshal(allhooks) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking"}`))) return } resp.WriteHeader(200) resp.Write(newjson) } //func deployWebhookCloudrun(ctx context.Context) { // service, err := cloudrun.NewService(ctx) // _ = err // // projectsLocationsService := cloudrun.NewProjectsLocationsService(service) // log.Printf("%#v", projectsLocationsService) // projectsLocationsGetCall := projectsLocationsService.Get("webhook") // log.Printf("%#v", projectsLocationsGetCall) // // location, err := projectsLocationsGetCall.Do() // log.Printf("%#v, err: %s", location, err) // // //func NewProjectsLocationsService(s *Service) *ProjectsLocationsService { // //func (r *ProjectsLocationsService) Get(name string) *ProjectsLocationsGetCall { // //func (c *ProjectsLocationsGetCall) Do(opts ...googleapi.CallOption) (*Location, error) { //} // Finds available ports func findAvailablePorts(startRange int64, endRange int64) string { for i := startRange; i < endRange; i++ { s := strconv.FormatInt(i, 10) l, err := net.Listen("tcp", ":"+s) if err == nil { l.Close() return s } } return "" } func handleSendalert(resp http.ResponseWriter, request *http.Request) { user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getworkflows: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if user.Role != "mail" && user.Role != "admin" { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "You don't have access to send mail"}`)) return } // ReferenceExecution and below are for execution continuations when user inputs arrive type mailcheck struct { Targets []string `json:"targets"` Body string `json:"body"` Subject string `json:"subject"` Type string `json:"type"` SenderCompany string `json:"sender_company"` ReferenceExecution string `json:"reference_execution"` WorkflowId string `json:"workflow_id"` ExecutionType string `json:"execution_type"` Start string `json:"start"` } body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Body data error on mail: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } var mailbody mailcheck err = json.Unmarshal(body, &mailbody) if err != nil { log.Printf("Unmarshal error on mail: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } ctx := context.Background() confirmMessage := ` You have a new alert from shuffler.io! %s Please contact us at shuffler.io or frikky@shuffler.io if there is an issue with this message.` parsedBody := fmt.Sprintf(confirmMessage, mailbody.Body) // FIXME - Make a continuation email here - might need more info from worker // making the request, e.g. what the next start-node is and execution_id for // how to make the links if mailbody.Type == "User input" { authkey := uuid.NewV4().String() log.Printf("Should handle differentiator for user input in email!") log.Printf("%#v", mailbody) url := "https://shuffler.io" //url := "http://localhost:5001" continueUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=true", url, mailbody.WorkflowId, authkey, mailbody.Start, mailbody.ReferenceExecution) stopUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute?authorization=%s&start=%s&reference_execution=%s&answer=false", url, mailbody.WorkflowId, authkey, mailbody.Start, mailbody.ReferenceExecution) //item := &memcache.Item{ // Key: authkey, // Value: []byte(fmt.Sprintf(`{"role": "workflow_%s"}`, mailbody.WorkflowId)), // Expiration: time.Minute * 1200, //} //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { // if err := memcache.Set(ctx, item); err != nil { // log.Printf("Error setting new user item: %v", err) // } //} else if err != nil { // log.Printf("error adding item: %v", err) //} else { // log.Printf("Set cache for %s", item.Key) //} parsedBody = fmt.Sprintf(` Action required! %s If this is TRUE click this: %s IF THIS IS FALSE, click this: %s Please contact us at shuffler.io or frikky@shuffler.io if there is an issue with this message. `, mailbody.Body, continueUrl, stopUrl) } msg := &mail.Message{ Sender: "Shuffle ", To: mailbody.Targets, Subject: fmt.Sprintf("Shuffle - %s - %s", mailbody.Type, mailbody.Subject), Body: parsedBody, } log.Println(msg.Body) if err := mail.Send(ctx, msg); err != nil { log.Printf("Couldn't send email: %v", err) } resp.WriteHeader(200) resp.Write([]byte("OK")) } func setBadMemcache(ctx context.Context, path string) { // Add to cache if it doesn't exist //item := &memcache.Item{ // Key: path, // Value: []byte(`{"success": false}`), // Expiration: time.Minute * 60, //} //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { // if err := memcache.Set(ctx, item); err != nil { // log.Printf("Error setting item: %v", err) // } //} else if err != nil { // log.Printf("error adding item: %v", err) //} else { // log.Printf("Set cache for %s", item.Key) //} } type Result struct { Success bool `json:"success"` Reason string `json:"reason"` List []string `json:"list"` } var docs_list = Result{List: []string{}} func getDocList(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } ctx := context.Background() //if item, err := memcache.Get(ctx, "docs_list"); err == memcache.ErrCacheMiss { // // Not in cache //} else if err != nil { // // Error with cache // log.Printf("Error getting item: %v", err) //} else { // resp.WriteHeader(200) // resp.Write([]byte(item.Value)) // return //} if len(docs_list.List) > 0 { b, err := json.Marshal(docs_list) if err != nil { log.Printf("Failed marshaling result: %s", err) //http.Error(resp, err.Error(), 500) } else { resp.WriteHeader(200) resp.Write(b) return } } client := github.NewClient(nil) _, item1, _, err := client.Repositories.GetContents(ctx, "frikky", "shuffle-docs", "docs", nil) if err != nil { log.Printf("Github error: %s", err) resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error listing directory: %s"`, err))) return } if len(item1) == 0 { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."`))) return } names := []string{} for _, item := range item1 { if !strings.HasSuffix(*item.Name, "md") { continue } names = append(names, (*item.Name)[0:len(*item.Name)-3]) } log.Println(names) var result Result result.Success = true result.Reason = "Success" result.List = names docs_list = result b, err := json.Marshal(result) if err != nil { http.Error(resp, err.Error(), 500) return } //item := &memcache.Item{ // Key: "docs_list", // Value: b, // Expiration: time.Minute * 60, //} //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { // if err := memcache.Set(ctx, item); err != nil { // log.Printf("Error setting item: %v", err) // } //} else if err != nil { // log.Printf("error adding item: %v", err) //} else { // log.Printf("Set cache for %s", item.Key) //} resp.WriteHeader(200) resp.Write(b) } // r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") var alldocs = map[string][]byte{} func getDocs(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") if len(location) != 5 { resp.WriteHeader(404) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) return } //ctx := context.Background() docPath := fmt.Sprintf("https://raw.githubusercontent.com/shaffuru/shuffle-docs/master/docs/%s.md", location[4]) //location[4] //var, ok := alldocs["asd"] key, ok := alldocs[fmt.Sprintf("%s", location[4])] // Custom cache for github issues lol if ok { resp.WriteHeader(200) resp.Write(key) return } client := &http.Client{} req, err := http.NewRequest( "GET", docPath, nil, ) if err != nil { resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) resp.WriteHeader(404) //setBadMemcache(ctx, docPath) return } newresp, err := client.Do(req) if err != nil { resp.WriteHeader(404) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) //setBadMemcache(ctx, docPath) return } body, err := ioutil.ReadAll(newresp.Body) if err != nil { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"`))) //setBadMemcache(ctx, docPath) return } type Result struct { Success bool `json:"success"` Reason string `json:"reason"` } var result Result result.Success = true //applog.Infof(ctx, string(body)) //applog.Infof(ctx, "Url: %s", docPath) //applog.Infof(ctx, "Status: %d", newresp.StatusCode) //applog.Infof(ctx, "GOT BODY OF LENGTH %d", len(string(body))) result.Reason = string(body) b, err := json.Marshal(result) if err != nil { http.Error(resp, err.Error(), 500) //setBadMemcache(ctx, docPath) return } alldocs[location[4]] = b // Add to cache if it doesn't exist //item := &memcache.Item{ // Key: docPath, // Value: b, // Expiration: time.Minute * 60, //} //if err := memcache.Add(ctx, item); err == memcache.ErrNotStored { // if err := memcache.Set(ctx, item); err != nil { // log.Printf("Error setting item: %v", err) // } //} else if err != nil { // log.Printf("error adding item: %v", err) //} else { // log.Printf("Set cache for %s", item.Key) //} resp.WriteHeader(200) resp.Write(b) } type OutlookProfile struct { OdataContext string `json:"@odata.context"` BusinessPhones []string `json:"businessPhones"` DisplayName string `json:"displayName"` GivenName string `json:"givenName"` JobTitle interface{} `json:"jobTitle"` Mail string `json:"mail"` MobilePhone interface{} `json:"mobilePhone"` OfficeLocation interface{} `json:"officeLocation"` PreferredLanguage interface{} `json:"preferredLanguage"` Surname string `json:"surname"` UserPrincipalName string `json:"userPrincipalName"` ID string `json:"id"` } type OutlookFolder struct { ID string `json:"id"` DisplayName string `json:"displayName"` ParentFolderID string `json:"parentFolderId"` ChildFolderCount int `json:"childFolderCount"` UnreadItemCount int `json:"unreadItemCount"` TotalItemCount int `json:"totalItemCount"` } type OutlookFolders struct { OdataContext string `json:"@odata.context"` OdataNextLink string `json:"@odata.nextLink"` Value []OutlookFolder `json:"value"` } func getOutlookFolders(client *http.Client) (OutlookFolders, error) { requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/frikky@shuffletest.onmicrosoft.com/mailfolders") ret, err := client.Get(requestUrl) if err != nil { log.Printf("FolderErr: %s", err) return OutlookFolders{}, err } if ret.StatusCode != 200 { log.Printf("Status folders: %d", ret.StatusCode) return OutlookFolders{}, err } body, err := ioutil.ReadAll(ret.Body) if err != nil { log.Printf("Body: %s", err) return OutlookFolders{}, err } //log.Printf("Body: %s", string(body)) mailfolders := OutlookFolders{} err = json.Unmarshal(body, &mailfolders) if err != nil { log.Printf("Unmarshal: %s", err) return OutlookFolders{}, err } //fmt.Printf("%#v", mailfolders) // FIXME - recursion for subfolders // Recursive struct // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) //for _, folder := range mailfolders.Value { // log.Println(folder.DisplayName) //} return mailfolders, nil } func getOutlookProfile(client *http.Client) (OutlookProfile, error) { requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/me?$select=mail") ret, err := client.Get(requestUrl) if err != nil { log.Printf("FolderErr: %s", err) return OutlookProfile{}, err } log.Printf("Status folders: %d", ret.StatusCode) body, err := ioutil.ReadAll(ret.Body) if err != nil { log.Printf("Body: %s", err) return OutlookProfile{}, err } profile := OutlookProfile{} err = json.Unmarshal(body, &profile) if err != nil { log.Printf("Unmarshal: %s", err) return OutlookProfile{}, err } return profile, nil } func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { code := request.URL.Query().Get("code") if len(code) == 0 { log.Println("No code") resp.WriteHeader(401) return } url := fmt.Sprintf("http://%s%s", request.Host, request.URL.EscapedPath()) log.Println(url) ctx := context.Background() client, accessToken, err := getOutlookClient(ctx, code, OauthToken{}, url) if err != nil { log.Printf("Oauth client failure - outlook register: %s", err) resp.WriteHeader(401) return } // This should be possible, and will also give the actual username profile, err := getOutlookProfile(client) if err != nil { log.Printf("Outlook profile failure: %s", err) resp.WriteHeader(401) return } // This is a state workaround, which should really be for CSRF checks lol state := request.URL.Query().Get("state") if len(state) == 0 { log.Println("No state") resp.WriteHeader(401) return } stateitems := strings.Split(state, "%26") if len(stateitems) == 1 { stateitems = strings.Split(state, "&") } // FIXME - trigger auth senderUser := "" trigger := TriggerAuth{} for _, item := range stateitems { itemsplit := strings.Split(item, "%3D") if len(itemsplit) == 1 { itemsplit = strings.Split(item, "=") } if len(itemsplit) != 2 { continue } // Do something here if itemsplit[0] == "workflow_id" { trigger.WorkflowId = itemsplit[1] } else if itemsplit[0] == "trigger_id" { trigger.Id = itemsplit[1] } else if itemsplit[0] == "type" { trigger.Type = itemsplit[1] } else if itemsplit[0] == "username" { trigger.Username = itemsplit[1] trigger.Owner = itemsplit[1] senderUser = itemsplit[1] } } // THis is an override based on the user in oauth return trigger.Username = profile.Mail trigger.Code = code trigger.OauthToken = OauthToken{ AccessToken: accessToken.AccessToken, TokenType: accessToken.TokenType, RefreshToken: accessToken.RefreshToken, Expiry: accessToken.Expiry, } //log.Printf("%#v", trigger) if trigger.WorkflowId == "" || trigger.Id == "" || trigger.Username == "" || trigger.Type == "" { log.Printf("All oauth items need to contain data to register a new state") resp.WriteHeader(401) return } // Should also update the user Userdata, err := getUser(ctx, senderUser) if err != nil { log.Printf("Username %s doesn't exist (oauth2): %s", trigger.Username, err) resp.WriteHeader(401) return } Userdata.Authentication = append(Userdata.Authentication, UserAuth{ Name: "Outlook", Description: "oauth2", Workflows: []string{trigger.WorkflowId}, Username: trigger.Username, Fields: []UserAuthField{ UserAuthField{ Key: "trigger_id", Value: trigger.Id, }, UserAuthField{ Key: "username", Value: trigger.Username, }, UserAuthField{ Key: "code", Value: code, }, UserAuthField{ Key: "type", Value: trigger.Type, }, }, }) // Set apikey for the user if they don't have one if len(Userdata.ApiKey) == 0 { newUser, err := generateApikey(ctx, *Userdata) Userdata = &newUser if err != nil { log.Printf("Failed to generate apikey for user %s when creating outlook sub: %s", Userdata.Username, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } } //err = setUser(Userdata) //if err != nil { // log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) // resp.WriteHeader(401) // return //} err = setTriggerAuth(ctx, trigger) if err != nil { log.Printf("Failed to set trigger auth for %s - %s", trigger.Username, err) resp.WriteHeader(401) return } // FIXME - not sure if these are good at all :) environmentVariables := map[string]string{ "FUNCTION_APIKEY": Userdata.ApiKey, "CALLBACKURL": "https://shuffler.io", "WORKFLOW_ID": trigger.WorkflowId, "TRIGGER_ID": trigger.Id, } applocation := fmt.Sprintf("gs://%s/triggers/outlooktrigger.zip", bucketName) hookname := fmt.Sprintf("outlooktrigger_%s", trigger.Id) err = deployCloudFunctionGo(ctx, hookname, defaultLocation, applocation, environmentVariables) if err != nil { log.Printf("Error deploying hook: %s", err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Issue with starting hook. Please wait a second and try again"}`))) return } resp.WriteHeader(200) resp.Write([]byte("OK")) } type OauthToken struct { AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` TokenType string `json:"TokenType" datastore:"TokenType,noindex"` RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` } type TriggerAuth struct { Id string `json:"id" datastore:"id"` SubscriptionId string `json:"subscriptionId" datastore:"subscriptionId"` Username string `json:"username" datastore:"username,noindex"` WorkflowId string `json:"workflow_id" datastore:"workflow_id,noindex"` Owner string `json:"owner" datastore:"owner"` Type string `json:"type" datastore:"type"` Code string `json:"code,omitempty" datastore:"code,noindex"` OauthToken OauthToken `json:"oauth_token,omitempty" datastore:"oauth_token"` } func getTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) { key := datastore.NameKey("trigger_auth", strings.ToLower(id), nil) triggerauth := &TriggerAuth{} if err := dbclient.Get(ctx, key, triggerauth); err != nil { return &TriggerAuth{}, err } return triggerauth, nil } func setTriggerAuth(ctx context.Context, trigger TriggerAuth) error { key1 := datastore.NameKey("trigger_auth", strings.ToLower(trigger.Id), nil) // New struct, to not add body, author etc if _, err := dbclient.Put(ctx, key1, &trigger); err != nil { log.Printf("Error adding trigger auth: %s", err) return err } return nil } // THis all of a sudden became really horrible.. fml func getOutlookClient(ctx context.Context, code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { conf := &oauth2.Config{ ClientID: "70e37005-c954-4290-b573-d4b94e484336", ClientSecret: ".eNw/A[kQFB5zL.agvRputdEJENeJ392", Scopes: []string{ "Mail.Read", "User.Read", }, RedirectURL: redirectUri, Endpoint: oauth2.Endpoint{ AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", TokenURL: "https://login.microsoftonline.com/common/oauth2/token", }, } if len(code) > 0 { access_token, err := conf.Exchange(ctx, code) if err != nil { log.Printf("Access_token issue: %s", err) return &http.Client{}, access_token, err } client := conf.Client(ctx, access_token) return client, access_token, nil } else { // Manually recreate the oauthtoken access_token := &oauth2.Token{ AccessToken: accessToken.AccessToken, TokenType: accessToken.TokenType, RefreshToken: accessToken.RefreshToken, Expiry: accessToken.Expiry, } client := conf.Client(ctx, access_token) return client, access_token, nil } } func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Exchange every time hmm // FIXME // Should really just get the code from the trigger that's being used OR the user triggerId := request.URL.Query().Get("trigger_id") if len(triggerId) == 0 { log.Println("No trigger_id supplied") resp.WriteHeader(401) return } ctx := context.Background() trigger, err := getTriggerAuth(ctx, triggerId) if err != nil { log.Printf("Trigger %s doesn't exist - outlook folders.", triggerId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Trigger doesn't exist."}`)) return } // FIXME - should be shuffler in literally every case except testing lol redirectDomain := "shuffler.io" url := fmt.Sprintf("https://%s/functions/outlook/register", redirectDomain) outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) if err != nil { log.Printf("Oauth client failure - outlook folders: %s", err) resp.WriteHeader(401) return } folders, err := getOutlookFolders(outlookClient) if err != nil { resp.WriteHeader(401) return } b, err := json.Marshal(folders.Value) if err != nil { log.Println("Failed to marshal folderdata") resp.WriteHeader(401) return } resp.WriteHeader(200) resp.Write(b) } func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } _, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } location := strings.Split(request.URL.String(), "/") var statsId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } statsId = location[4] } ctx := context.Background() statisticsId := "global_statistics" nameKey := statsId key := datastore.NameKey(statisticsId, nameKey, nil) statisticsItem := StatisticsItem{} if err := dbclient.Get(ctx, key, &statisticsItem); err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } b, err := json.Marshal(statisticsItem) if err != nil { log.Println("Failed to marshal data: %s", err) resp.WriteHeader(401) return } resp.WriteHeader(200) resp.Write([]byte(b)) } func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } if strings.Contains(workflowId, "?") { workflowId = strings.Split(workflowId, "?")[0] } ctx := context.Background() trigger, err := getTriggerAuth(ctx, workflowId) if err != nil { log.Printf("Trigger %s doesn't exist - specific trigger.", workflowId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } if user.Username != trigger.Owner && user.Role != "admin" { log.Printf("Wrong user (%s) for trigger %s", user.Username, trigger.Id) resp.WriteHeader(401) return } trigger.OauthToken = OauthToken{} trigger.Code = "" b, err := json.Marshal(trigger) if err != nil { log.Println("Failed to marshal data") resp.WriteHeader(401) return } resp.WriteHeader(200) resp.Write(b) } func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string var triggerId string if location[1] == "api" { if len(location) <= 6 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] triggerId = location[6] } if len(workflowId) == 0 || len(triggerId) == 0 { log.Printf("Ids can't be zero when deleting %s", workflowId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } ctx := context.Background() workflow, err := getWorkflow(ctx, workflowId) if err != nil { log.Printf("Failed getting the workflow locally: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in outlook deploy: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - have a check for org etc too.. if user.Id != workflow.Owner && user.Role != "admin" { log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // Check what kind of sub it is err = handleOutlookSubRemoval(ctx, workflowId, triggerId) if err != nil { log.Printf("Failed sub removal: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string) error { // DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions/%s", subscriptionId) req, err := http.NewRequest( "DELETE", fullUrl, nil, ) req.Header.Add("Content-Type", "application/json") res, err := outlookClient.Do(req) if err != nil { log.Printf("Client: %s", err) return err } if res.StatusCode != 200 && res.StatusCode != 201 && res.StatusCode != 204 { return errors.New(fmt.Sprintf("Bad status code when deleting subscription: %d", res.StatusCode)) } body, err := ioutil.ReadAll(res.Body) if err != nil { log.Printf("Body: %s", err) return err } _ = body return nil } // Remove AUTH // Remove function // Remove subscription func handleOutlookSubRemoval(ctx context.Context, workflowId, triggerId string) error { // 1. Get the auth for trigger // 2. Stop the subscription // 3. Remove the function // 4. Remove the database entry for auth trigger, err := getTriggerAuth(ctx, triggerId) if err != nil { log.Printf("Trigger auth %s doesn't exist - outlook sub removal.", triggerId) return err } url := fmt.Sprintf("https://shuffler.io") outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) if err != nil { log.Printf("Oauth client failure - triggerauth sub removal: %s", err) return err } notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, trigger.Id) curSubscriptions, err := getOutlookSubscriptions(outlookClient) if err == nil { for _, sub := range curSubscriptions.Value { if sub.NotificationURL == notificationURL { log.Printf("Removing existing subscription %s", sub.Id) removeOutlookSubscription(outlookClient, sub.Id) } } } else { log.Printf("Failed to get subscriptions - need to overwrite") } // FIXME - not removing the function, as the trigger still exists //err = removeOutlookTriggerFunction(triggerId) //if err != nil { // return err //} return nil } // This sets up the sub with outlook itself // Parses data from the workflow to see whether access is right to subscribe it // Creates the cloud function for outlook return // Wait for it to be available, then schedule a workflow to it func createOutlookSub(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } location := strings.Split(request.URL.String(), "/") var workflowId string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } workflowId = location[4] } ctx := context.Background() workflow, err := getWorkflow(ctx, workflowId) if err != nil { log.Printf("Failed getting the workflow locally: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in outlook deploy: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - have a check for org etc too.. if user.Id != workflow.Owner && user.Role != "admin" { log.Printf("Wrong user (%s) for workflow %s when deploying outlook", user.Username, workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Println("Handle outlook subscription for trigger") // Should already be authorized at this point, as the workflow is shared body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Failed body read for workflow %s", workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Println(string(body)) // Based on the input data from frontend type CurTrigger struct { Name string `json:"name"` Folders []string `json:"folders"` ID string `json:"id"` } var curTrigger CurTrigger err = json.Unmarshal(body, &curTrigger) if err != nil { log.Printf("Failed body read unmarshal for trigger %s", workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } if len(curTrigger.Folders) == 0 { log.Printf("Error for %s. Choosing folders is required, currently 0", workflow.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // Now that it's deployed - wait a few seconds before generating: // 1. Oauth2 token thingies for outlook.office.com // 2. Set the url to have the right mailboxes (probably ID?) ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") // 3. Set the callback URL to be the new trigger // 4. Run subscription test // 5. Set the subscriptionId to the trigger object // First - lets regenerate an oauth token for outlook.office.com from the original items trigger, err := getTriggerAuth(ctx, curTrigger.ID) if err != nil { log.Printf("Trigger %s doesn't exist - outlook sub.", curTrigger.ID) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": ""}`)) return } // url doesn't really matter here url := fmt.Sprintf("https://shuffler.io") outlookClient, _, err := getOutlookClient(ctx, "", trigger.OauthToken, url) if err != nil { log.Printf("Oauth client failure - triggerauth: %s", err) resp.WriteHeader(401) return } // Location + notificationURL := fmt.Sprintf("https://%s-%s.cloudfunctions.net/outlooktrigger_%s", defaultLocation, gceProject, curTrigger.ID) log.Println(notificationURL) // This is here simply to let the function start // Usually takes 10 attempts minimum :O // 10 * 5 = 50 seconds. That's waaay too much :( //notificationURL = "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_e2ce43b0-997e-4980-9617-6eadbc68cf88" //notificationURL = "https://de4fc12b.ngrok.io" curSubscriptions, err := getOutlookSubscriptions(outlookClient) if err == nil { for _, sub := range curSubscriptions.Value { if sub.NotificationURL == notificationURL { log.Printf("Removing existing subscription %s", sub.Id) removeOutlookSubscription(outlookClient, sub.Id) } } } else { log.Printf("Failed to get subscriptions - need to overwrite") } maxFails := 15 failCnt := 0 log.Println(curTrigger.Folders) for { subId, err := makeOutlookSubscription(outlookClient, curTrigger.Folders, notificationURL) if err != nil { failCnt += 1 log.Printf("Failed making oauth subscription, retrying in 5 seconds: %s", err) time.Sleep(5 * time.Second) if failCnt == maxFails { log.Printf("Failed to set up subscription %d times.", maxFails) resp.WriteHeader(401) return } continue } // Set the ID somewhere here trigger.SubscriptionId = subId err = setTriggerAuth(ctx, *trigger) if err != nil { log.Printf("Failed setting triggerauth: %s", err) } break } log.Printf("Successfully handled outlook subscription for trigger %s in workflow %s", curTrigger.ID, workflow.ID) //log.Printf("%#v", user) resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } // Lists the users current subscriptions func getOutlookSubscriptions(outlookClient *http.Client) (SubscriptionsWrapper, error) { fullUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/subscriptions") req, err := http.NewRequest( "GET", fullUrl, nil, ) req.Header.Add("Content-Type", "application/json") res, err := outlookClient.Do(req) if err != nil { log.Printf("suberror Client: %s", err) return SubscriptionsWrapper{}, err } body, err := ioutil.ReadAll(res.Body) if err != nil { log.Printf("Suberror Body: %s", err) return SubscriptionsWrapper{}, err } newSubs := SubscriptionsWrapper{} err = json.Unmarshal(body, &newSubs) if err != nil { return SubscriptionsWrapper{}, err } return newSubs, nil } type SubscriptionsWrapper struct { OdataContext string `json:"@odata.context"` Value []Subscription `json:"value"` } type Subscription struct { ChangeType string `json:"changeType"` NotificationURL string `json:"notificationUrl"` Resource string `json:"resource"` ExpirationDateTime string `json:"expirationDateTime"` ClientState string `json:"clientState"` Id string `json:"id"` } func makeOutlookSubscription(client *http.Client, folderIds []string, notificationURL string) (string, error) { fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" // FIXME - this expires rofl t := time.Now().Local().Add(time.Minute * time.Duration(4300)) timeFormat := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d.0000000Z", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) log.Println(timeFormat) resource := fmt.Sprintf("me/mailfolders('%s')/messages", strings.Join(folderIds, "','")) log.Println(resource) sub := Subscription{ ChangeType: "created", NotificationURL: notificationURL, ExpirationDateTime: timeFormat, ClientState: "This is a test", Resource: resource, } data, err := json.Marshal(sub) if err != nil { log.Printf("Marshal: %s", err) return "", err } req, err := http.NewRequest( "POST", fullUrl, bytes.NewBuffer(data), ) req.Header.Add("Content-Type", "application/json") res, err := client.Do(req) if err != nil { log.Printf("Client: %s", err) return "", err } log.Printf("Status: %d", res.StatusCode) body, err := ioutil.ReadAll(res.Body) if err != nil { log.Printf("Body: %s", err) return "", err } if res.StatusCode != 200 && res.StatusCode != 201 { return "", errors.New(fmt.Sprintf("Subscription failed: %s", string(body))) } // Use data from body here to create thingy newSub := Subscription{} err = json.Unmarshal(body, &newSub) if err != nil { return "", err } return newSub.Id, nil } func getOpenapi(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Just here to verify that the user is logged in _, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } location := strings.Split(request.URL.String(), "/") var id string if location[1] == "api" { if len(location) <= 4 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } id = location[4] } if len(id) != 32 { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME - FIX AUTH WITH APP ctx := context.Background() //_, err = getApp(ctx, id) //if err == nil { // log.Println("You're supposed to be able to continue now.") //} parsedApi, err := getOpenApiDatastore(ctx, id) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } parsedApi.Success = true data, err := json.Marshal(parsedApi) if err != nil { resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling parsed swagger: %s"}`, err))) return } resp.WriteHeader(200) resp.Write(data) } func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Just here to verify that the user is logged in _, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("Bodyreader err: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) return } newbody := string(body) newbody = strings.TrimSpace(newbody) if strings.HasPrefix(newbody, "\"") { newbody = newbody[1:len(newbody)] } if strings.HasSuffix(newbody, "\"") { newbody = newbody[0 : len(newbody)-1] } req, err := http.NewRequest("GET", newbody, nil) if err != nil { log.Printf("Requestbuilder err: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed building request"}`)) return } httpClient := &http.Client{} newresp, err := httpClient.Do(req) if err != nil { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed making request for data"`))) return } defer newresp.Body.Close() urlbody, err := ioutil.ReadAll(newresp.Body) if err != nil { resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"`))) return } resp.WriteHeader(200) resp.Write(urlbody) } func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { type versionCheck struct { Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"` OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"` } //body = []byte(`swagger: "2.0"`) //body = []byte(`swagger: '1.0'`) //newbody := string(body) //newbody = strings.TrimSpace(newbody) //body = []byte(newbody) //log.Println(string(body)) //tmpbody, err := yaml.YAMLToJSON(body) //log.Println(err) //log.Println(string(tmpbody)) // This has to be done in a weird way because Datastore doesn't // support map[string]interface and similar (openapi3.Swagger) var version versionCheck parsed := ParsedOpenApi{} swaggerdata := []byte{} idstring := "" isJson := false err := json.Unmarshal(body, &version) if err != nil { //log.Printf("Json err: %s", err) err = yaml.Unmarshal(body, &version) if err != nil { log.Printf("Yaml error: %s", err) } else { //log.Printf("Successfully parsed YAML!") } } else { isJson = true log.Printf("Successfully parsed JSON!") } if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { version.Swagger = version.SwaggerVersion } if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { //log.Println("Handling v3 API") swaggerv3, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) if err != nil { return ParsedOpenApi{}, err } swaggerdata, err = json.Marshal(swaggerv3) if err != nil { log.Printf("Failed unmarshaling v3 data: %s", err) return ParsedOpenApi{}, err } hasher := md5.New() hasher.Write(swaggerdata) idstring = hex.EncodeToString(hasher.Sum(nil)) } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { // Convert //log.Println("Handling v2 API") var swagger openapi2.Swagger //log.Println(string(body)) err = json.Unmarshal(body, &swagger) if err != nil { //log.Printf("Json error? %s", err) err = gyaml.Unmarshal(body, &swagger) if err != nil { log.Printf("Yaml error: %s", err) return ParsedOpenApi{}, err } else { //log.Printf("Valid yaml!") } } swaggerv3, err := openapi2conv.ToV3Swagger(&swagger) if err != nil { log.Printf("Failed converting from openapi2 to 3: %s", err) return ParsedOpenApi{}, err } swaggerdata, err = json.Marshal(swaggerv3) if err != nil { log.Printf("Failed unmarshaling v3 data: %s", err) return ParsedOpenApi{}, err } hasher := md5.New() hasher.Write(swaggerdata) idstring = hex.EncodeToString(hasher.Sum(nil)) } if len(swaggerdata) > 0 { body = swaggerdata } // Overwrite with new json data _ = isJson body = swaggerdata // Parsing it to swagger 3 parsed = ParsedOpenApi{ ID: idstring, Body: string(body), Success: true, } return parsed, err } // FIXME: Migrate this to use handleSwaggerValidation() func validateSwagger(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } // Just here to verify that the user is logged in _, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } body, err := ioutil.ReadAll(request.Body) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) return } type versionCheck struct { Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"` OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"` } //body = []byte(`swagger: "2.0"`) //body = []byte(`swagger: '1.0'`) //newbody := string(body) //newbody = strings.TrimSpace(newbody) //body = []byte(newbody) //log.Println(string(body)) //tmpbody, err := yaml.YAMLToJSON(body) //log.Println(err) //log.Println(string(tmpbody)) // This has to be done in a weird way because Datastore doesn't // support map[string]interface and similar (openapi3.Swagger) var version versionCheck isJson := false err = json.Unmarshal(body, &version) if err != nil { log.Printf("Json err: %s", err) err = yaml.Unmarshal(body, &version) if err != nil { log.Printf("Yaml error: %s", err) //resp.WriteHeader(422) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi to json and yaml: %s"}`, err))) //return } else { log.Printf("Successfully parsed YAML!") } } else { isJson = true log.Printf("Successfully parsed JSON!") } if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { version.Swagger = version.SwaggerVersion } if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { log.Println("Handling v3 API") swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) if err != nil { resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } hasher := md5.New() hasher.Write(body) idstring := hex.EncodeToString(hasher.Sum(nil)) log.Printf("Swagger v3 validation success with ID %s!", idstring) log.Printf("Paths: %d", len(swagger.Paths)) if !isJson { log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) } parsed := ParsedOpenApi{ ID: idstring, Body: string(body), } ctx := context.Background() err = setOpenApiDatastore(ctx, idstring, parsed) if err != nil { log.Printf("Failed uploading openapi to datastore: %s", err) resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err))) return } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, idstring))) return } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { // Convert log.Println("Handling v2 API") var swagger openapi2.Swagger //log.Println(string(body)) err = json.Unmarshal(body, &swagger) if err != nil { log.Printf("Json error? %s", err) err = gyaml.Unmarshal(body, &swagger) if err != nil { log.Printf("Yaml error: %s", err) } resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err))) return } swaggerv3, err := openapi2conv.ToV3Swagger(&swagger) if err != nil { log.Printf("Failed converting from openapi2 to 3: %s", err) resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed converting from openapi2 to openapi3: %s"}`, err))) return } swaggerdata, err := json.Marshal(swaggerv3) if err != nil { log.Printf("Failed unmarshaling v3 data: %s", err) resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed marshalling swaggerv3 data: %s"}`, err))) return } hasher := md5.New() hasher.Write(swaggerdata) idstring := hex.EncodeToString(hasher.Sum(nil)) if !isJson { log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring) } log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring) parsed := ParsedOpenApi{ ID: idstring, Body: string(swaggerdata), } ctx := context.Background() err = setOpenApiDatastore(ctx, idstring, parsed) if err != nil { log.Printf("Failed uploading openapi2 to datastore: %s", err) resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed reading openapi2: %s"}`, err))) return } resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, idstring))) return } /* else { log.Printf("Swagger / OpenAPI version %s is not supported or there is an error.", version.Swagger) resp.WriteHeader(422) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Swagger version %s is not currently supported"}`, version.Swagger))) return } */ // save the openapi ID resp.WriteHeader(422) resp.Write([]byte(`{"success": false}`)) } // Creates an app from the app builder func verifySwagger(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } user, err := handleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in verify swagger: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } body, err := ioutil.ReadAll(request.Body) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`)) return } type Test struct { Editing bool `datastore:"editing"` Id string `datastore:"id"` Image string `datastore:"image"` } var test Test err = json.Unmarshal(body, &test) if err != nil { log.Printf("Failed unmarshalling test: %s", err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // Get an identifier hasher := md5.New() hasher.Write(body) newmd5 := hex.EncodeToString(hasher.Sum(nil)) if test.Editing { // Quick verification test ctx := context.Background() app, err := getApp(ctx, test.Id) if err != nil { log.Printf("Error getting app when editing: %s", app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } // FIXME: Check whether it's in use. if user.Id != app.Owner && user.Role != "admin" { log.Printf("Wrong user (%s) for app %s when verifying swagger", user.Username, app.Name) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } log.Printf("EDITING APP WITH ID %s", app.ID) newmd5 = app.ID } // Generate new app integration (bump version) // Test = client side with fetch? ctx := context.Background() //client, err := storage.NewClient(ctx) //if err != nil { // log.Printf("Failed to create client (storage): %v", err) // resp.WriteHeader(401) // resp.Write([]byte(`{"success": false, "reason": "Failed creating client"}`)) // return //} swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) if err != nil { log.Printf("Swagger validation error: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed verifying openapi"}`)) return } if strings.Contains(swagger.Info.Title, " ") { strings.Replace(swagger.Info.Title, " ", "", -1) } basePath, err := buildStructure(swagger, newmd5) if err != nil { log.Printf("Failed to build base structure: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed building baseline structure"}`)) return } //log.Printf("Should generate yaml") api, pythonfunctions, err := generateYaml(swagger, newmd5) if err != nil { log.Printf("Failed building and generating yaml: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed building and parsing yaml"}`)) return } api.Owner = user.Id err = dumpApi(basePath, api) if err != nil { log.Printf("Failed dumping yaml: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed dumping yaml"}`)) return } identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, newmd5) classname := strings.Replace(identifier, " ", "", -1) classname = strings.Replace(classname, "-", "", -1) parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) if err != nil { log.Printf("Failed dumping python: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed dumping appcode"}`)) return } identifier = strings.Replace(identifier, " ", "-", -1) identifier = strings.Replace(identifier, "_", "-", -1) log.Printf("Successfully parsed %s. Proceeding to docker container", identifier) // Now that the baseline is setup, we need to make it into a cloud function // 1. Upload the API to datastore for use // 2. Get code from baseline/app_base.py & baseline/static_baseline.py // 3. Stitch code together from these two + our new app // 4. Zip the folder to cloud storage // 5. Upload as cloud function // 1. Upload the API to datastore err = deployAppToDatastore(ctx, api) if err != nil { log.Printf("Failed adding app to db: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed adding app to db"}`)) return } // 2. Get all the required code appbase, staticBaseline, err := getAppbase() if err != nil { log.Printf("Failed getting appbase: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed getting appbase code"}`)) return } // Have to do some quick checks of the python code (: _, parsedCode = formatAppfile(parsedCode) fixedAppbase := fixAppbase(appbase) runner := getRunner(classname) // 2. Put it together stitched := string(staticBaseline) + strings.Join(fixedAppbase, "\n") + parsedCode + string(runner) //log.Println(stitched) // 3. Zip and stream it directly in the directory _, err = streamZipdata(ctx, identifier, stitched, "requests\nurllib3") if err != nil { log.Printf("Zipfile error: %s", err) resp.WriteHeader(500) resp.Write([]byte(`{"success": false, "reason": "Failed to build zipfile"}`)) return } log.Printf("Successfully uploaded ZIPFILE for %s", identifier) // 4. Upload as cloud function - this apikey is specifically for cloud functions rofl //environmentVariables := map[string]string{ // "FUNCTION_APIKEY": apikey, //} //fullLocation := fmt.Sprintf("gs://%s/%s", bucketName, applocation) //err = deployCloudFunctionPython(ctx, identifier, defaultLocation, fullLocation, environmentVariables) //if err != nil { // log.Printf("Error uploading cloud function: %s", err) // resp.WriteHeader(500) // resp.Write([]byte(`{"success": false, "reason": "Failed to upload function"}`)) // return //} // 4. Build the image locally. // FIXME: Should be moved to a local docker registry dockerLocation := fmt.Sprintf("%s/Dockerfile", basePath) log.Printf("Dockerfile: %s", dockerLocation) versionName := fmt.Sprintf("%s_%s", strings.ReplaceAll(api.Name, " ", "-"), api.AppVersion) dockerTags := []string{ fmt.Sprintf("%s:%s", baseDockerName, identifier), fmt.Sprintf("%s:%s", baseDockerName, versionName), } err = buildImage(dockerTags, dockerLocation) if err != nil { log.Printf("Docker build error: %s", err) resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Error in Docker build"}`))) return } found := false foundNumber := 0 log.Printf("Checking for api with ID %s", newmd5) for appCounter, app := range user.PrivateApps { if app.ID == api.ID { found = true foundNumber = appCounter break } else if app.Name == api.Name && app.AppVersion == api.AppVersion { found = true foundNumber = appCounter break } else if app.PrivateID == test.Id && test.Editing { found = true foundNumber = appCounter break } } // Updating the user with the new app so that it can easily be retrieved if !found { user.PrivateApps = append(user.PrivateApps, api) } else { user.PrivateApps[foundNumber] = api } err = setUser(ctx, &user) if err != nil { log.Printf("Failed adding verification for user %s: %s", user.Username, err) resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "Failed updating user"}`))) return } parsed := ParsedOpenApi{ ID: api.ID, Body: string(body), } setOpenApiDatastore(ctx, api.ID, parsed) err = increaseStatisticsField(ctx, "total_apps_created", api.ID, 1) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } err = increaseStatisticsField(ctx, "openapi_apps_created", api.ID, 1) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } resp.WriteHeader(200) resp.Write([]byte(`{"success": true}`)) } func healthCheckHandler(resp http.ResponseWriter, request *http.Request) { fmt.Fprint(resp, "OK") } func runInit(ctx context.Context) { // Setting stats for backend starts (failure count as well) err := increaseStatisticsField(ctx, "backend_executions", "", 1) if err != nil { log.Printf("Failed increasing local stats: %s", err) } // Gets environments and inits if it doesn't exist count, err := getEnvironmentCount() if count == 0 && err == nil { item := Environment{ Name: "Shuffle", Type: "onprem", } err = setEnvironment(ctx, &item) if err != nil { log.Printf("Failed setting up new environment") } } // Gets schedules and starts them schedules, err := getAllSchedules(ctx) if err != nil { log.Printf("Failed getting schedules during service init: %s", err) } else { log.Printf("Setting up %d schedule(s)", len(schedules)) for _, schedule := range schedules { job := func() { request := &http.Request{ Method: "POST", Body: ioutil.NopCloser(strings.NewReader(schedule.Argument)), } _, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request) if err != nil { log.Printf("Failed to execute: %s", err) } } //log.Printf("Schedule time: every %d seconds", schedule.Seconds) jobret, err := newscheduler.Every(schedule.Seconds).Seconds().NotImmediately().Run(job) if err != nil { log.Printf("Failed to schedule workflow: %s", err) } scheduledJobs[schedule.Id] = jobret } } // Getting apps to see if we should initialize a test workflowapps, err := getAllWorkflowApps(ctx) if err != nil { log.Printf("Failed getting apps: %s", err) } else if err == nil && len(workflowapps) == 0 { log.Printf("Downloading default workflow apps") fs := memfs.New() storer := memory.NewStorage() url := os.Getenv("APP_DOWNLOAD_LOCATION") if len(url) == 0 { url = "https://github.com/frikky/shuffle-apps" } username := os.Getenv("APP_DOWNLOAD_AUTH_USERNAME") password := os.Getenv("APP_DOWNLOAD_AUTH_PASSWORD") cloneOptions := &git.CloneOptions{ URL: url, } if len(username) > 0 && len(password) > 0 { cloneOptions.Auth = &http2.BasicAuth{ Username: username, Password: password, } } log.Printf("Getting apps from %s", url) r, err := git.Clone(storer, fs, cloneOptions) if err != nil { log.Printf("Failed loading repo into memory: %s", err) } dir, err := fs.ReadDir("") if err != nil { log.Printf("Failed reading folder: %s", err) } _ = r //iterateAppGithubFolders(fs, dir, "", "testing") // FIXME: Get all the apps? iterateAppGithubFolders(fs, dir, "", "") } log.Printf("Downloading OpenAPI data for search - EXTRA APPS") apis := "https://github.com/frikky/OpenAPI-security-definitions" // THis gets memory problems hahah //apis := "https://github.com/APIs-guru/openapi-directory" fs := memfs.New() storer := memory.NewStorage() cloneOptions := &git.CloneOptions{ URL: apis, } _, err = git.Clone(storer, fs, cloneOptions) if err != nil { log.Printf("Failed loading repo %s into memory: %s", err) } else { log.Printf("Finished git clone. Looking for updates to the repo.") dir, err := fs.ReadDir("") if err != nil { log.Printf("Failed reading folder: %s", err) } iterateOpenApiGithub(fs, dir, "", "") log.Printf("Finished downloading extra API samples") } log.Printf("Finished INIT") } func init() { var err error ctx := context.Background() log.Printf("Running INIT process") dbclient, err = datastore.NewClient(ctx, gceProject) if err != nil { log.Printf("DBclient error during init: %s", err) } go runInit(ctx) r := mux.NewRouter() r.HandleFunc("/api/v1/_ah/health", healthCheckHandler) // Sends an email if the right things are specified r.HandleFunc("/functions/sendmail", handleSendalert).Methods("POST", "OPTIONS") r.HandleFunc("/functions/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") r.HandleFunc("/functions/outlook/getFolders", handleGetOutlookFolders).Methods("GET", "OPTIONS") // General r.HandleFunc("/api/v1/login", handleLogin).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/logout", handleLogout).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/register", handleRegister).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/checkusers", checkAdminLogin).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getusers", handleGetUsers).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getenvironments", handleGetEnvironments).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/setenvironments", handleSetEnvironments).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/getsettings", handleSettings).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/generateapikey", handleApiGeneration).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/docs", getDocList).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") // Queuebuilder and Workflow streams. First is to update a stream, second to get a stream // Changed from workflows/streams to streams, as appengine was messing up // This does not increase the API counter r.HandleFunc("/api/v1/workflows/queue", handleGetWorkflowqueue).Methods("GET") r.HandleFunc("/api/v1/workflows/queue/confirm", handleGetWorkflowqueueConfirm).Methods("POST") r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST") r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") // App specific r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps", getWorkflowApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/apps/search", getSpecificApps).Methods("POST", "OPTIONS") // Legacy app things r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/apps", getWorkflowApps).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/apps", setNewWorkflowApp).Methods("PUT", "OPTIONS") // Workflows // FIXME - implement the queue counter lol /* Everything below here increases the counters*/ r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS") //r.HandleFunc("/api/v1/workflows/{key}/execute_fs", executeWorkflowFS) r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook", createOutlookSub).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/outlook/{triggerId}", handleDeleteOutlookSub).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions", getWorkflowExecutions).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/executions/{key}/abort", abortExecution).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", getSpecificWorkflow).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", saveWorkflow).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") // Triggers // Webhook redirect to the correct cloud function r.HandleFunc("/api/v1/hooks/new", handleNewHook).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/hooks/{key}", handleWebhookCallback).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/hooks/{key}/delete", handleDeleteHook).Methods("DELETE", "OPTIONS") // Trigger hmm r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS") // OpenAPI configuration r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/get_openapi_uri", echoOpenapiData).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/validate_openapi", validateSwagger).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/get_openapi/{key}", getOpenapi).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/execution_cleanup", cleanupExecutions).Methods("GET", "OPTIONS") http.Handle("/", r) } // Had to move away from mux, which means Method is fucked up right now. func main() { //init() hostname, err := os.Hostname() if err != nil { hostname = "MISSING" } innerPort := os.Getenv("BACKEND_PORT") if innerPort == "" { log.Printf("Running on %s:5001", hostname) log.Fatal(http.ListenAndServe(":5001", nil)) } else { log.Printf("Running on %s:%s", hostname, innerPort) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", innerPort), nil)) } }