You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1412 lines
45 KiB
1412 lines
45 KiB
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"log"
|
|
"math"
|
|
mathRand "math/rand"
|
|
"net"
|
|
"net/http"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.sequentialread.com/forest/greenhouse/pki"
|
|
errors "git.sequentialread.com/forest/pkg-errors"
|
|
base58 "github.com/shengdoushi/base58"
|
|
)
|
|
|
|
type BaseHTTPService struct {
|
|
client *http.Client
|
|
clientExpiry *time.Time
|
|
ClientFactory func() (*http.Client, *time.Time, error)
|
|
}
|
|
|
|
type BackendApp struct {
|
|
BaseHTTPService
|
|
WorkingDirectory string
|
|
EmailService *EmailService
|
|
Model *DBModel
|
|
DigitalOcean *DigitalOceanService
|
|
Gandi *GandiService
|
|
FreeSubdomainDomain string
|
|
BackblazeB2 *BackblazeB2Service
|
|
SSH *SSHService
|
|
ThresholdProvisioning *ThresholdProvisioningService
|
|
ThresholdManagementPort int
|
|
ThresholdPort int
|
|
AdminTenantId int
|
|
AdminThresholdNodeId string
|
|
GreenhouseListenPort int
|
|
GreenhouseThresholdServiceId string
|
|
}
|
|
|
|
type ThresholdMetrics struct {
|
|
InboundByTenant map[string]int64
|
|
OutboundByTenant map[string]int64
|
|
}
|
|
|
|
type VPSInstanceWithTenants struct {
|
|
TenantSettings map[string]TunnelSettings
|
|
VPSInstance *VPSInstance
|
|
}
|
|
|
|
type stringBytesTuple struct {
|
|
id string
|
|
bytes int64
|
|
}
|
|
type vpsInstanceBytesTuple struct {
|
|
instance *VPSInstance
|
|
bytes int64
|
|
}
|
|
type tenantMoveTuple struct {
|
|
tenantId int
|
|
instanceId string
|
|
bytes int64
|
|
}
|
|
type knapsackGuess struct {
|
|
moves []*tenantMoveTuple
|
|
distance int64
|
|
score int64
|
|
}
|
|
|
|
type taskResult struct {
|
|
Name string
|
|
Err error
|
|
Result interface{}
|
|
}
|
|
|
|
const tenantShufflingThreshold = float64(0.2)
|
|
const knapsackShufflingMinimizationFactor = float64(0.3)
|
|
const knapsackNumberOfGuessesFactor = float64(5)
|
|
const shufflingCircuitBreakerLimit = 1000
|
|
const GIGABYTE = int64(1000000000)
|
|
const TERABYTE = int64(1000000000000)
|
|
|
|
const managementClientCertSubject = "management@greenhouse.server.garden"
|
|
const freeSubdomainDomain = "greenhouseusers.com"
|
|
|
|
var projectedOverageAllowedBeforeSpawningNewInstance int64 = GIGABYTE * 250
|
|
var projectedUnderageAllowedBeforeTerminatingInstance int64 = TERABYTE
|
|
|
|
func initBackend(
|
|
workingDirectory string,
|
|
config *Config,
|
|
pkiService *pki.PKIService,
|
|
model *DBModel,
|
|
emailService *EmailService,
|
|
) *BackendApp {
|
|
|
|
adminThresholdNodeId := "greenhouse_node_id"
|
|
greenhouseThresholdServiceId := "greenhouse_https"
|
|
|
|
toReturn := BackendApp{
|
|
WorkingDirectory: workingDirectory,
|
|
EmailService: emailService,
|
|
Model: model,
|
|
DigitalOcean: NewDigitalOceanService(config),
|
|
Gandi: NewGandiService(config, freeSubdomainDomain),
|
|
FreeSubdomainDomain: freeSubdomainDomain,
|
|
BackblazeB2: NewBackblazeB2Service(config),
|
|
SSH: NewSSHService(config),
|
|
ThresholdProvisioning: NewThresholdProvisioningService(config, pkiService, config.AdminTenantId, adminThresholdNodeId, greenhouseThresholdServiceId),
|
|
ThresholdPort: config.ThresholdPort,
|
|
ThresholdManagementPort: config.ThresholdManagementPort,
|
|
AdminTenantId: config.AdminTenantId,
|
|
AdminThresholdNodeId: adminThresholdNodeId,
|
|
GreenhouseListenPort: config.FrontendPort,
|
|
GreenhouseThresholdServiceId: greenhouseThresholdServiceId,
|
|
}
|
|
|
|
// TODO move this BaseHTTPService into the ThresholdProvisioning service?
|
|
toReturn.ClientFactory = func() (*http.Client, *time.Time, error) {
|
|
|
|
caCert, err := pkiService.GetCACertificate(mainCAName)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
expiryTime := time.Now().Add(time.Hour * 24)
|
|
cert, err := pkiService.GetClientTLSCertificate(managementAPIAuthCAName, managementClientCertSubject, expiryTime)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
caCertPool := x509.NewCertPool()
|
|
caCertPool.AddCert(caCert)
|
|
|
|
tlsClientConfig := &tls.Config{
|
|
Certificates: []tls.Certificate{cert},
|
|
RootCAs: caCertPool,
|
|
}
|
|
tlsClientConfig.BuildNameToCertificate()
|
|
|
|
return &http.Client{
|
|
Transport: &http.Transport{
|
|
TLSClientConfig: tlsClientConfig,
|
|
},
|
|
Timeout: 10 * time.Second,
|
|
}, &expiryTime, nil
|
|
}
|
|
|
|
return &toReturn
|
|
}
|
|
|
|
func (app *BackendApp) InitializeTenant(tenantId int, email string) error {
|
|
emailSplit := strings.Split(email, "@")
|
|
if len(emailSplit) != 2 {
|
|
return errors.Errorf("InitializeTenant(): invalid email '%s'", email)
|
|
}
|
|
emailDomainSplit := strings.Split(emailSplit[1], ".")
|
|
if len(emailDomainSplit) < 2 {
|
|
return errors.Errorf("InitializeTenant(): invalid email '%s'", email)
|
|
}
|
|
replaceInvalidCharactersRegex := regexp.MustCompile("[^a-z0-9]")
|
|
replaceRepeatedHyphensRegex := regexp.MustCompile("-+")
|
|
makeSubdomain := func(input string) string {
|
|
toReturn := replaceInvalidCharactersRegex.ReplaceAllString(input, "-")
|
|
toReturn = replaceRepeatedHyphensRegex.ReplaceAllString(toReturn, "-")
|
|
return strings.Trim(toReturn, "-")
|
|
}
|
|
|
|
subdomain := makeSubdomain(emailSplit[0])
|
|
var err error
|
|
alreadyTaken := true
|
|
i := 0
|
|
for alreadyTaken && i < 1000 {
|
|
i++
|
|
alreadyTaken, err = app.Model.SetFreeSubdomain(tenantId, subdomain)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if i == 1 {
|
|
subdomain = makeSubdomain(fmt.Sprintf("%s-%s", emailSplit[0], emailDomainSplit[0]))
|
|
} else if i == 2 {
|
|
subdomain = makeSubdomain(email)
|
|
} else {
|
|
subdomain = makeSubdomain(fmt.Sprintf("%s-%d", emailSplit[0], i-2))
|
|
}
|
|
}
|
|
if i >= 1000 {
|
|
return errors.Errorf("InitializeTenant(): something went wrong generating a subdomain for '%s'", email)
|
|
}
|
|
|
|
start, end, bucket, err := app.Model.GetNextReservedPorts()
|
|
if err != nil {
|
|
return errors.Wrapf(err, "InitializeTenant() for '%s':", email)
|
|
}
|
|
|
|
err = app.Model.SetReservedPorts(tenantId, start, end, bucket)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "InitializeTenant() for '%s':", email)
|
|
}
|
|
|
|
apiTokenBuffer := make([]byte, 16)
|
|
rand.Read(apiTokenBuffer)
|
|
apiToken := base58.Encode(apiTokenBuffer, base58.BitcoinAlphabet)
|
|
rawHash := sha256.Sum256([]byte(apiToken))
|
|
hashedAPIToken := fmt.Sprintf("%x", rawHash)
|
|
err = app.Model.SetHashedAPIToken(tenantId, hashedAPIToken)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "InitializeTenant() for '%s':", email)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (app *BackendApp) ConsumeMetrics() error {
|
|
|
|
vpsInstances, err := app.Model.GetVPSInstances()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
billingYear, billingMonth, _, _, _ := getBillingTimeInfo()
|
|
|
|
actions := make([]func() taskResult, len(vpsInstances))
|
|
i := 0
|
|
for instanceId, instance := range vpsInstances {
|
|
|
|
actions[i] = func() taskResult {
|
|
// TODO move the metrics to the management API / to the client side in threshold
|
|
responseBytes, err := app.MyHTTP200(
|
|
"GET",
|
|
fmt.Sprintf("https://%s:%d/consumeMetrics", instance.IPV4, app.ThresholdManagementPort),
|
|
nil,
|
|
nil,
|
|
)
|
|
//log.Println(string(responseBytes))
|
|
result := ThresholdMetrics{}
|
|
if err == nil {
|
|
err = json.Unmarshal(responseBytes, &result)
|
|
}
|
|
if err == nil {
|
|
err = app.Model.RecordVPSUsage(instance, result, billingYear, billingMonth)
|
|
}
|
|
return taskResult{
|
|
Name: instanceId,
|
|
Err: err,
|
|
Result: result,
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
|
|
results := doInParallel(false, actions...)
|
|
|
|
errorStrings := []string{}
|
|
bytesByTenant := map[int]int64{}
|
|
for _, result := range results {
|
|
if result.Err != nil {
|
|
errorStrings = append(errorStrings, fmt.Sprintf("error collecting metrics from %s: %+v", result.Name, result.Err))
|
|
} else {
|
|
instanceBytesByTenant := result.Result.(ThresholdMetrics)
|
|
for tenantIdString, bytez := range instanceBytesByTenant.InboundByTenant {
|
|
tenantId, err := strconv.Atoi(tenantIdString)
|
|
if err != nil {
|
|
errorStrings = append(errorStrings, fmt.Sprintf("can't convert tenantId '%s' to integer: %s", tenantIdString, err))
|
|
} else {
|
|
bytesByTenant[tenantId] += bytez
|
|
}
|
|
}
|
|
for tenantIdString, bytez := range instanceBytesByTenant.OutboundByTenant {
|
|
tenantId, err := strconv.Atoi(tenantIdString)
|
|
if err != nil {
|
|
errorStrings = append(errorStrings, fmt.Sprintf("can't convert tenantId '%s' to integer: %s", tenantIdString, err))
|
|
} else {
|
|
bytesByTenant[tenantId] += bytez
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if len(errorStrings) > 0 {
|
|
return errors.Errorf("ConsumeMetrics() doInParallel(actions...): %s\n\n", strings.Join(errorStrings, "\n"))
|
|
}
|
|
|
|
err = app.Model.RecordTenantsUsage(bytesByTenant)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (app *BackendApp) GetInstances() (map[string]*VPSInstance, map[string]*VPSInstance, map[string]*VPSInstance, error) {
|
|
|
|
dbVPSInstances, err := app.Model.GetVPSInstances()
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
cloudVPSInstances, err := app.DigitalOcean.List()
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
|
|
// jsonBytes, err = json.MarshalIndent(dbVPSInstances, "", " ")
|
|
// log.Printf("dbVPSInstances: %s\n\n", string(jsonBytes))
|
|
|
|
// jsonBytes, err = json.MarshalIndent(digitalOceanVpsInstances, "", " ")
|
|
// log.Printf("digitalOceanVpsInstances: %s\n\n", string(jsonBytes))
|
|
|
|
validVpsInstances := map[string]*VPSInstance{}
|
|
onlyCloudInstances := map[string]*VPSInstance{}
|
|
onlyDBInstances := map[string]*VPSInstance{}
|
|
for k, v := range cloudVPSInstances {
|
|
if _, has := dbVPSInstances[k]; !has {
|
|
onlyCloudInstances[k] = v
|
|
}
|
|
}
|
|
for k, v := range dbVPSInstances {
|
|
if _, has := cloudVPSInstances[k]; !has {
|
|
onlyDBInstances[k] = v
|
|
} else if !v.Deleted && !v.Deprecated {
|
|
validVpsInstances[k] = v
|
|
}
|
|
}
|
|
return validVpsInstances, onlyDBInstances, onlyCloudInstances, nil
|
|
}
|
|
|
|
func (app *BackendApp) HealthcheckInstances(vpsInstances map[string]*VPSInstance) map[string]bool {
|
|
if len(vpsInstances) == 0 {
|
|
return map[string]bool{}
|
|
}
|
|
|
|
actions := make([]func() taskResult, len(vpsInstances))
|
|
i := 0
|
|
for instanceId, instance := range vpsInstances {
|
|
actions[i] = func() taskResult {
|
|
_, err := app.MyHTTP200(
|
|
"GET",
|
|
fmt.Sprintf("https://%s:%d/ping", instance.IPV4, app.ThresholdManagementPort),
|
|
nil,
|
|
nil,
|
|
)
|
|
return taskResult{
|
|
Name: instanceId,
|
|
Err: err,
|
|
Result: nil,
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
|
|
results := doInParallel(false, actions...)
|
|
|
|
healthStatus := map[string]bool{}
|
|
for _, result := range results {
|
|
if result.Err != nil {
|
|
log.Printf("error pinging %s: %s", result.Name, result.Err)
|
|
}
|
|
healthStatus[result.Name] = (result.Err == nil)
|
|
}
|
|
|
|
return healthStatus
|
|
}
|
|
|
|
func getBillingTimeInfo() (int, int, time.Time, time.Time, float64) {
|
|
billingYear := time.Now().UTC().Year()
|
|
billingMonth := int(time.Now().UTC().Month())
|
|
|
|
nextBillingYear := billingYear
|
|
nextBillingMonth := billingMonth + 1
|
|
if nextBillingMonth > 12 {
|
|
nextBillingMonth = 1
|
|
nextBillingYear++
|
|
}
|
|
utcLocation, err := time.LoadLocation("UTC")
|
|
if err != nil {
|
|
err = errors.Wrap(err, "can't calculate billing because can't load UTC location. This should have been caught at startup")
|
|
log.Printf("%+v\n", err)
|
|
panic(err)
|
|
}
|
|
startOfBillingMonth := time.Date(billingYear, time.Month(billingMonth), 1, 0, 0, 0, 0, utcLocation)
|
|
endOfBillingMonth := time.Date(nextBillingYear, time.Month(nextBillingMonth), 0, 23, 59, 59, int(int64(time.Second))-1, utcLocation)
|
|
//TODO
|
|
monthDuration := float64(int64(endOfBillingMonth.Sub(startOfBillingMonth)))
|
|
monthElapsed := float64(int64(endOfBillingMonth.Sub(time.Now().UTC())))
|
|
amountOfMonthElapsed := monthElapsed / monthDuration
|
|
|
|
return billingMonth, billingYear, startOfBillingMonth, endOfBillingMonth, amountOfMonthElapsed
|
|
}
|
|
|
|
func (app *BackendApp) Rebalance() (bool, error) {
|
|
|
|
billingYear, billingMonth, _, _, amountOfMonthElapsed := getBillingTimeInfo()
|
|
|
|
//TODO
|
|
desiredInstancesPerTenant := 2
|
|
tenantPinDuration := 6 * time.Hour
|
|
|
|
tenants, err := app.Model.GetTenants()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
tenantVpsInstanceRows, err := app.Model.GetTenantVPSInstanceRows(billingYear, billingMonth)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
validVpsInstances, onlyDBInstances, onlyCloudInstances, err := app.GetInstances()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
errorStrings := []string{}
|
|
for _, v := range onlyDBInstances {
|
|
errorStrings = append(errorStrings, fmt.Sprintf("instance %s (%s) is in the database, but not in the provider api", v.GetId(), v.IPV4))
|
|
}
|
|
for _, v := range onlyCloudInstances {
|
|
errorStrings = append(errorStrings, fmt.Sprintf("instance %s (%s) is in the provider api, but not in the database", v.GetId(), v.IPV4))
|
|
}
|
|
|
|
// issue warnings for inconsistencies between DB and provider
|
|
if len(errorStrings) > 0 {
|
|
return false, errors.Errorf("VPS instances are inconsistent: \n%s\n", strings.Join(errorStrings, "\n"))
|
|
}
|
|
|
|
// aggregate dedicated vps instances count onto tenants
|
|
keysToRemove := []string{}
|
|
for k, vpsInstance := range validVpsInstances {
|
|
if vpsInstance.TenantId != 0 {
|
|
keysToRemove = append(keysToRemove, k)
|
|
tenants[vpsInstance.TenantId].DedicatedVPSCount += 1
|
|
}
|
|
}
|
|
// TODO do we need to do anything else with these besides cutting them from the "validVpsInstances" ?
|
|
for _, k := range keysToRemove {
|
|
delete(validVpsInstances, k)
|
|
}
|
|
|
|
// jsonBytes, err := json.MarshalIndent(tenants, "", " ")
|
|
// log.Printf("tenants: %s\n\n", string(jsonBytes))
|
|
|
|
// jsonBytes, err = json.MarshalIndent(tenantVpsInstanceRows, "", " ")
|
|
// log.Printf("tenantVpsInstanceRows: %s\n\n", string(jsonBytes))
|
|
|
|
// jsonBytes, err = json.MarshalIndent(validVpsInstances, "", " ")
|
|
// log.Printf("validVpsInstances: %s\n\n", string(jsonBytes))
|
|
|
|
// aggregate the bandwidth usage from the tenantVPS relation onto both
|
|
// note that the total bandwidth on the tenants may be larger
|
|
// than the total bandwidth on the vps instances because some vps instances may have been deleted
|
|
|
|
// if you update the following loop, consider updating the similar one in frontend.go admin handler 😬
|
|
tenantAllocationCount := map[int]int{}
|
|
shadowConfigs := map[string]map[int]*TunnelSettings{}
|
|
workingAllocations := map[string]map[int]bool{}
|
|
originalAllocations := map[string]map[int]bool{}
|
|
pinned := map[string]map[int]bool{}
|
|
for _, row := range tenantVpsInstanceRows {
|
|
vpsInstanceId := row.GetVPSInstanceId()
|
|
vpsInstance, hasVPSInstance := validVpsInstances[vpsInstanceId]
|
|
tenant, hasTenant := tenants[row.TenantId]
|
|
if hasVPSInstance {
|
|
vpsInstance.Bytes += row.Bytes
|
|
}
|
|
if hasTenant {
|
|
tenant.Bytes += row.Bytes
|
|
}
|
|
if hasTenant && hasVPSInstance {
|
|
_, has := shadowConfigs[vpsInstanceId]
|
|
if !has {
|
|
shadowConfigs[vpsInstanceId] = map[int]*TunnelSettings{}
|
|
workingAllocations[vpsInstanceId] = map[int]bool{}
|
|
originalAllocations[vpsInstanceId] = map[int]bool{}
|
|
}
|
|
|
|
shadowConfigs[vpsInstanceId][tenant.Id] = row.ShadowConfig
|
|
|
|
// if the tenannts allocation to this instance was removed recently (less than tenantPinDuration ago)
|
|
// then add the allocation to pinned, keeping the allocation live for a while while DNS updates propagate.
|
|
if row.DeactivatedAt != nil && row.DeactivatedAt.Add(tenantPinDuration).After(time.Now()) {
|
|
pinned[vpsInstanceId][tenant.Id] = true
|
|
}
|
|
|
|
if row.Active {
|
|
workingAllocations[vpsInstanceId][tenant.Id] = true
|
|
originalAllocations[vpsInstanceId][tenant.Id] = true
|
|
tenantAllocationCount[row.TenantId]++
|
|
}
|
|
}
|
|
}
|
|
|
|
// Deactiveate tenants who have surpassed thier service limit for this billing month
|
|
bytesPerCent := 1000000000
|
|
for _, tenant := range tenants {
|
|
bandwidthCostCents := int(tenant.Bytes / int64(bytesPerCent))
|
|
if bandwidthCostCents >= tenant.ServiceLimitCents {
|
|
tenant.Deactivated = true
|
|
}
|
|
}
|
|
|
|
// -------------------------------------------------------------------------------
|
|
// STEP 1: scale the # of instances up or down depending on total projected usage.
|
|
|
|
totalUsageSoFar := int64(0)
|
|
totalUsageByActiveTenantsSoFar := int64(0)
|
|
for _, tenant := range tenants {
|
|
totalUsageSoFar += tenant.Bytes
|
|
if !tenant.Deactivated {
|
|
totalUsageByActiveTenantsSoFar += tenant.Bytes
|
|
}
|
|
}
|
|
|
|
soFarFloat := float64(totalUsageSoFar)
|
|
activeTenantsSoFarFloat := float64(totalUsageByActiveTenantsSoFar)
|
|
totalProjectedUsageFloat := soFarFloat + ((activeTenantsSoFarFloat / amountOfMonthElapsed) - activeTenantsSoFarFloat)
|
|
//totalProjectedUsage := int64(totalProjectedUsageFloat)
|
|
|
|
totalMonthlyAllotment := int64(0)
|
|
for _, vpsInstance := range validVpsInstances {
|
|
if !vpsInstance.Deprecated && !vpsInstance.Deleted {
|
|
// TODO handle BytesMonthly when node is created in the middle of the month / near end of month
|
|
amountOfMonthThatThisInstanceWillHaveBeenTurnedOnFor := float64(1)
|
|
totalMonthlyAllotment += int64(float64(vpsInstance.BytesMonthly) * amountOfMonthThatThisInstanceWillHaveBeenTurnedOnFor)
|
|
}
|
|
}
|
|
totalMonthlyAllotmentFloat := float64(totalMonthlyAllotment)
|
|
|
|
overageFloat := totalProjectedUsageFloat - totalMonthlyAllotmentFloat
|
|
overflowAmountFloat := overageFloat - float64(projectedOverageAllowedBeforeSpawningNewInstance)
|
|
if overflowAmountFloat > 0 || len(validVpsInstances) == 0 {
|
|
instancesToCreate := int(math.Ceil(overflowAmountFloat / float64(DEFAULT_INSTANCE_MONTHLY_BYTES)))
|
|
if instancesToCreate < 1 && len(validVpsInstances) == 0 {
|
|
instancesToCreate = 1
|
|
}
|
|
log.Printf("spawning %d instances...", instancesToCreate)
|
|
|
|
tasks := []func() taskResult{}
|
|
for i := 0; i < instancesToCreate; i++ {
|
|
tasks = append(tasks, func() taskResult {
|
|
instance, err := app.SpawnNewMultitenantInstance()
|
|
return taskResult{
|
|
Err: err,
|
|
Result: instance,
|
|
Name: strconv.Itoa(i),
|
|
}
|
|
})
|
|
}
|
|
results := doInParallel(false, tasks...)
|
|
|
|
errors := []string{}
|
|
for _, result := range results {
|
|
if result.Err != nil {
|
|
errors = append(errors, fmt.Sprintf("%+v", result.Err))
|
|
}
|
|
}
|
|
if len(errors) > 0 {
|
|
return false, fmt.Errorf("SpawnNewMultitenantInstance failed: \n%s\n", strings.Join(errors, "\n"))
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
underusageAmount := -overageFloat - float64(projectedUnderageAllowedBeforeTerminatingInstance)
|
|
if underusageAmount > 0 && len(validVpsInstances) > 1 {
|
|
instancesToDeprecate := int(math.Ceil(underusageAmount / float64(DEFAULT_INSTANCE_MONTHLY_BYTES)))
|
|
if instancesToDeprecate > len(validVpsInstances)-1 {
|
|
instancesToDeprecate = len(validVpsInstances) - 1
|
|
}
|
|
log.Printf("deprecating %d instances...", instancesToDeprecate)
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// STEP 2: shuffle tenants around to balance load.
|
|
// if we got to this point that means we didn't spawn or deprecate any instances this time around.
|
|
|
|
// sanity check the desiredInstancesPerTenant
|
|
if desiredInstancesPerTenant > len(validVpsInstances) {
|
|
desiredInstancesPerTenant = len(validVpsInstances)
|
|
}
|
|
|
|
// the allocate function will ADD (not remove) the tenant to the <instanceCount> vps instances that don't already
|
|
// have that tenant on them & have the lowest projected usage for this month.
|
|
allocate := func(tenant *TenantInfo, instanceCount int, workingAllocations *map[string]map[int]bool) {
|
|
lowestUsedInstances := []stringBytesTuple{}
|
|
|
|
for _, instance := range validVpsInstances {
|
|
// get total projected usage for each instance
|
|
// un-allocate any existing allocations for this tenant.. ?
|
|
// find lowest desiredInstancesPerTenant instances & allocate them
|
|
|
|
instanceAllocations, has := (*workingAllocations)[instance.GetId()]
|
|
// if this instance already has this tenant allocated, skip it.
|
|
if has && instanceAllocations[tenant.Id] || instance.Deprecated || instance.Deleted {
|
|
continue
|
|
}
|
|
|
|
instanceMetadata := stringBytesTuple{
|
|
id: instance.GetId(),
|
|
bytes: getInstanceProjectedUsage(instance, workingAllocations, &tenants, desiredInstancesPerTenant, amountOfMonthElapsed),
|
|
}
|
|
if len(lowestUsedInstances) < instanceCount {
|
|
lowestUsedInstances = append(lowestUsedInstances, instanceMetadata)
|
|
} else {
|
|
indexOfMostUsed := -1
|
|
for i, instance := range lowestUsedInstances {
|
|
if indexOfMostUsed == -1 || instance.bytes > lowestUsedInstances[indexOfMostUsed].bytes {
|
|
indexOfMostUsed = i
|
|
}
|
|
}
|
|
if instanceMetadata.bytes < lowestUsedInstances[indexOfMostUsed].bytes {
|
|
lowestUsedInstances[indexOfMostUsed] = instanceMetadata
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, instanceMetadata := range lowestUsedInstances {
|
|
if _, has := (*workingAllocations)[instanceMetadata.id]; !has {
|
|
(*workingAllocations)[instanceMetadata.id] = map[int]bool{}
|
|
}
|
|
(*workingAllocations)[instanceMetadata.id][tenant.Id] = true
|
|
}
|
|
}
|
|
|
|
// 2.1 allocate tenants who don't have the desired # of active vps allocations yet.
|
|
for _, tenant := range tenants {
|
|
if !tenant.Deactivated && tenantAllocationCount[tenant.Id] < desiredInstancesPerTenant {
|
|
allocate(tenant, desiredInstancesPerTenant-tenantAllocationCount[tenant.Id], &workingAllocations)
|
|
}
|
|
}
|
|
|
|
// 2.2 shuffle tenant allocations around so tenants
|
|
// are moved from instances with predicted overages to instances with predicted underages
|
|
|
|
iterations := 0
|
|
doneShuffling := false
|
|
for !doneShuffling && iterations < shufflingCircuitBreakerLimit && len(validVpsInstances) > 1 {
|
|
highestSurplus := vpsInstanceBytesTuple{instance: nil, bytes: 0}
|
|
lowestSurplus := vpsInstanceBytesTuple{instance: nil, bytes: 0}
|
|
|
|
for _, instance := range validVpsInstances {
|
|
if !instance.Deprecated && !instance.Deleted {
|
|
//TODO
|
|
amountOfMonthThatThisInstanceWillHaveBeenTurnedOnFor := float64(1)
|
|
// TODO handle BytesMonthly when node is created in the middle of the month / near end of month
|
|
totalMonthlyAllotment := int64(float64(instance.BytesMonthly) * amountOfMonthThatThisInstanceWillHaveBeenTurnedOnFor)
|
|
surplus := totalMonthlyAllotment - getInstanceProjectedUsage(
|
|
instance,
|
|
&workingAllocations,
|
|
&tenants,
|
|
desiredInstancesPerTenant,
|
|
amountOfMonthElapsed,
|
|
)
|
|
|
|
if highestSurplus.instance == nil || highestSurplus.bytes < surplus {
|
|
highestSurplus = vpsInstanceBytesTuple{instance: instance, bytes: surplus}
|
|
}
|
|
if lowestSurplus.instance == nil || lowestSurplus.bytes > surplus {
|
|
lowestSurplus = vpsInstanceBytesTuple{instance: instance, bytes: surplus}
|
|
}
|
|
}
|
|
}
|
|
|
|
// if there are no instances which have a predicted overage, exit.
|
|
if lowestSurplus.bytes >= 0 {
|
|
doneShuffling = true
|
|
break
|
|
}
|
|
|
|
// lets say that the most overused instance has a predicted overusage of 100gb
|
|
// and the most underused instance has a predicted underusage of 700gb
|
|
// the difference between those is 800gb.
|
|
// so we want to move 400gb (half the difference) of estimated future traffic
|
|
// from the most overused to the most underused.
|
|
// if we did that, then the result would be balanced: both predicted to be underused by 300gb
|
|
desiredDifference := int64Abs(lowestSurplus.bytes-highestSurplus.bytes) / 2
|
|
|
|
// only continue shuffling if the difference between the two instances
|
|
// is over 20% of thier monthly allowance
|
|
// TODO handle BytesMonthly when node is created in the middle of the month / near end of month
|
|
averageMonthlyBytes := float64(highestSurplus.instance.BytesMonthly+lowestSurplus.instance.BytesMonthly) * float64(0.5)
|
|
if float64(desiredDifference) < averageMonthlyBytes*tenantShufflingThreshold {
|
|
doneShuffling = true
|
|
break
|
|
}
|
|
|
|
tenantsOfEmptiestInstance := workingAllocations[highestSurplus.instance.GetId()]
|
|
tenantsOfFullestInstance := workingAllocations[lowestSurplus.instance.GetId()]
|
|
|
|
// we are going to create a list of tenantId, instanceId pairs, one for each
|
|
// of the tenant allocations on each of the most underused and overused instances
|
|
// each list item also includes the net effect on the projected usage difference of the two instances
|
|
// assuming that the tenant was moved from one instance to the other
|
|
|
|
effectsOfMovingTenantToOtherInstance := []tenantMoveTuple{}
|
|
|
|
for tenantId := range tenantsOfEmptiestInstance {
|
|
tenantUsageShare := float64(tenants[tenantId].Bytes / int64(desiredInstancesPerTenant))
|
|
futureUsage := (tenantUsageShare / amountOfMonthElapsed) - tenantUsageShare
|
|
// only consider moving the tenant if it does not already exist on the destination
|
|
if !tenantsOfFullestInstance[tenantId] {
|
|
effectsOfMovingTenantToOtherInstance = append(
|
|
effectsOfMovingTenantToOtherInstance,
|
|
tenantMoveTuple{
|
|
tenantId: tenantId,
|
|
instanceId: highestSurplus.instance.GetId(),
|
|
bytes: int64(futureUsage * 2),
|
|
},
|
|
)
|
|
}
|
|
}
|
|
for tenantId := range tenantsOfFullestInstance {
|
|
tenantUsageShare := float64(tenants[tenantId].Bytes / int64(desiredInstancesPerTenant))
|
|
futureUsage := (tenantUsageShare / amountOfMonthElapsed) - tenantUsageShare
|
|
// only consider moving the tenant if it does not already exist on the destination
|
|
if !tenantsOfEmptiestInstance[tenantId] {
|
|
effectsOfMovingTenantToOtherInstance = append(
|
|
effectsOfMovingTenantToOtherInstance,
|
|
tenantMoveTuple{
|
|
tenantId: tenantId,
|
|
instanceId: lowestSurplus.instance.GetId(),
|
|
bytes: int64(-futureUsage * 2),
|
|
},
|
|
)
|
|
}
|
|
}
|
|
|
|
// we constructed a list of all possible moves we could make to shuffle tenants between the two,
|
|
// now we use a heuristic method to find the best combination of moves which
|
|
// gets us closest to our desiredDifference.
|
|
// (weighted to also minimize the # of moves, based on knapsackShufflingMinimizationFactor)
|
|
// This is basically the knapsack problem: https://en.wikipedia.org/wiki/Knapsack_problem
|
|
|
|
positiveValue := int64(0)
|
|
negativeValue := int64(0)
|
|
for _, move := range effectsOfMovingTenantToOtherInstance {
|
|
if move.bytes > 0 {
|
|
positiveValue += move.bytes
|
|
} else {
|
|
negativeValue += move.bytes
|
|
}
|
|
}
|
|
|
|
bestGuessSoFar := knapsackGuess{
|
|
moves: []*tenantMoveTuple{},
|
|
distance: desiredDifference,
|
|
score: getKnapsackSolutionScore(desiredDifference, desiredDifference, 0),
|
|
}
|
|
numberOfAttempts := int(float64(len(effectsOfMovingTenantToOtherInstance)) * knapsackNumberOfGuessesFactor)
|
|
for attempt := 0; attempt < numberOfAttempts; attempt++ {
|
|
difference := int64(0)
|
|
moves := []*tenantMoveTuple{}
|
|
positiveTotal := positiveValue
|
|
negativeTotal := negativeValue
|
|
permutation := getRandomPermutation(len(effectsOfMovingTenantToOtherInstance))
|
|
for i := range effectsOfMovingTenantToOtherInstance {
|
|
index := permutation[i]
|
|
move := effectsOfMovingTenantToOtherInstance[index]
|
|
proposedDifference := difference + move.bytes
|
|
proposedDistance := int64Abs(proposedDifference - desiredDifference)
|
|
proposedScore := getKnapsackSolutionScore(desiredDifference, proposedDistance, len(moves)+1)
|
|
|
|
// if moving this tenant would push our current guess "too far" in the positive direction
|
|
if proposedDifference > desiredDifference {
|
|
|
|
// ok, we overshot... would it be possible to "walk it back"?
|
|
// or did we overshoot only a little bit & create a good solution?
|
|
|
|
impossibleToGoBack := proposedDifference+negativeTotal > desiredDifference
|
|
if impossibleToGoBack && proposedScore > bestGuessSoFar.score {
|
|
continue
|
|
}
|
|
}
|
|
// if moving this tenant would push our current guess "too far" in the negative direction
|
|
if proposedDifference < 0 {
|
|
impossibleToGoBack := proposedDifference+positiveTotal < 0
|
|
if impossibleToGoBack {
|
|
continue
|
|
}
|
|
}
|
|
|
|
difference = proposedDifference
|
|
moves = append(moves, &move)
|
|
if move.bytes > 0 {
|
|
positiveTotal -= move.bytes
|
|
} else {
|
|
negativeTotal -= move.bytes
|
|
}
|
|
if proposedScore < bestGuessSoFar.score {
|
|
bestGuessSoFar = knapsackGuess{
|
|
moves: moves,
|
|
distance: proposedDistance,
|
|
score: proposedScore,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(bestGuessSoFar.moves) == 0 {
|
|
doneShuffling = true
|
|
} else {
|
|
for _, move := range bestGuessSoFar.moves {
|
|
if move.instanceId == highestSurplus.instance.GetId() {
|
|
delete(workingAllocations[move.instanceId], move.tenantId)
|
|
workingAllocations[lowestSurplus.instance.GetId()][move.tenantId] = true
|
|
} else {
|
|
delete(workingAllocations[move.instanceId], move.tenantId)
|
|
workingAllocations[highestSurplus.instance.GetId()][move.tenantId] = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if iterations == shufflingCircuitBreakerLimit {
|
|
return false, fmt.Errorf(
|
|
`something went wrong shuffling tenants shufflingCircuitBreakerLimit was reached (%d iterations)`,
|
|
shufflingCircuitBreakerLimit,
|
|
)
|
|
}
|
|
|
|
// Step 3: Now that we have the workingAllocations shuffled to balance the load, we need to apply
|
|
// those changes to the instances.
|
|
|
|
// we store a "shadow config" in the database for the tenants membership on vps.
|
|
// Then when we are ready to apply a new configuration, we do a diff against the current "shadow config"
|
|
// and only make requests to each instance which has diffs.
|
|
|
|
existingConfig := map[string]map[int]*TunnelSettings{}
|
|
newConfig := map[string]map[int]*TunnelSettings{}
|
|
|
|
for instanceId, instanceAllocations := range originalAllocations {
|
|
for tenantId := range instanceAllocations {
|
|
settings, has := shadowConfigs[instanceId][tenantId]
|
|
if !has {
|
|
settings = &TunnelSettings{
|
|
AuthorizedDomains: []string{},
|
|
}
|
|
}
|
|
|
|
if _, has = existingConfig[instanceId]; !has {
|
|
existingConfig[instanceId] = map[int]*TunnelSettings{}
|
|
}
|
|
existingConfig[instanceId][tenantId] = settings
|
|
}
|
|
}
|
|
|
|
authorizeFreeSubdomain := func(tenant *TenantInfo) *TunnelSettings {
|
|
if tenant.Subdomain != "" {
|
|
tenant.TunnelSettings.AuthorizedDomains = append(
|
|
tenant.TunnelSettings.AuthorizedDomains,
|
|
fmt.Sprintf("%s.%s", tenant.Subdomain, app.FreeSubdomainDomain),
|
|
)
|
|
}
|
|
return tenant.TunnelSettings
|
|
}
|
|
|
|
// Add both workingAllocations and pinned to newConfig
|
|
for instanceId, instanceAllocations := range workingAllocations {
|
|
for tenantId := range instanceAllocations {
|
|
if _, has := newConfig[instanceId]; !has {
|
|
newConfig[instanceId] = map[int]*TunnelSettings{}
|
|
}
|
|
if !tenants[tenantId].Deactivated {
|
|
newConfig[instanceId][tenantId] = authorizeFreeSubdomain(tenants[tenantId])
|
|
}
|
|
}
|
|
}
|
|
for instanceId, instanceAllocations := range pinned {
|
|
for tenantId := range instanceAllocations {
|
|
if _, has := newConfig[instanceId]; !has {
|
|
newConfig[instanceId] = map[int]*TunnelSettings{}
|
|
}
|
|
if !tenants[tenantId].Deactivated {
|
|
newConfig[instanceId][tenantId] = authorizeFreeSubdomain(tenants[tenantId])
|
|
}
|
|
}
|
|
}
|
|
|
|
// go over the new and existing configs and mark any instanceIds which have differences
|
|
|
|
changedInstanceIds := map[string]bool{}
|
|
|
|
for instanceId, instanceTenants := range existingConfig {
|
|
if _, has := newConfig[instanceId]; !has {
|
|
changedInstanceIds[instanceId] = true
|
|
continue
|
|
}
|
|
newInstanceTenants := newConfig[instanceId]
|
|
for tenantId, tenantConfig := range instanceTenants {
|
|
newTenantConfig, has := newInstanceTenants[tenantId]
|
|
if !has || !newTenantConfig.DeepEquals(tenantConfig) {
|
|
changedInstanceIds[instanceId] = true
|
|
}
|
|
}
|
|
}
|
|
for instanceId, instanceTenants := range newConfig {
|
|
if _, has := existingConfig[instanceId]; !has {
|
|
changedInstanceIds[instanceId] = true
|
|
continue
|
|
}
|
|
existingInstanceTenants := newConfig[instanceId]
|
|
for tenantId, tenantConfig := range instanceTenants {
|
|
existingTenantConfig, has := existingInstanceTenants[tenantId]
|
|
// TODO remove hardcoded true
|
|
if !has || !existingTenantConfig.DeepEquals(tenantConfig) || true {
|
|
changedInstanceIds[instanceId] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
// we send PUT requests to all the instances that have config changes
|
|
|
|
actions := make([]func() taskResult, len(changedInstanceIds))
|
|
i := 0
|
|
for instanceId := range changedInstanceIds {
|
|
instance := validVpsInstances[instanceId]
|
|
|
|
actions[i] = func() taskResult {
|
|
err := app.saveVpsInstanceTenantSettings(billingYear, billingMonth, instance, newConfig[instanceId])
|
|
return taskResult{
|
|
Name: instanceId,
|
|
Err: err,
|
|
Result: nil,
|
|
}
|
|
}
|
|
i++
|
|
}
|
|
|
|
log.Println("saving tenants assignments to threshold servers...")
|
|
|
|
results := doInParallel(false, actions...)
|
|
|
|
for _, result := range results {
|
|
if result.Err != nil {
|
|
return true, result.Err
|
|
}
|
|
}
|
|
|
|
freeSubdomains := map[string][]string{}
|
|
for instanceId, tenantIds := range workingAllocations {
|
|
instance, hasInstance := validVpsInstances[instanceId]
|
|
if hasInstance {
|
|
for tenantId := range tenantIds {
|
|
tenant, hasTenant := tenants[tenantId]
|
|
if hasTenant && tenant.Subdomain != "" {
|
|
_, hasIps := freeSubdomains[tenant.Subdomain]
|
|
if !hasIps {
|
|
freeSubdomains[tenant.Subdomain] = []string{instance.IPV4}
|
|
} else {
|
|
freeSubdomains[tenant.Subdomain] = append(freeSubdomains[tenant.Subdomain], instance.IPV4)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
log.Println("updating tenants free subdomains...")
|
|
|
|
err = app.Gandi.UpdateFreeSubdomains(freeSubdomains)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
|
|
log.Println("rebalance complete!")
|
|
|
|
return true, nil
|
|
}
|
|
|
|
func (app *BackendApp) WriteAdminTenantThresholdConfig() error {
|
|
tenants, err := app.Model.GetTenants()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for tenantId, tenant := range tenants {
|
|
if tenantId == app.AdminTenantId {
|
|
clientConfig, err := app.ThresholdProvisioning.GetClientConfig(
|
|
app.AdminTenantId, fmt.Sprintf("%s.%s", tenant.Subdomain, app.FreeSubdomainDomain), "greenhouse",
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
clientConfigBytes, err := json.MarshalIndent(clientConfig, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = ioutil.WriteFile("./threshold/config.json", clientConfigBytes, 755)
|
|
|
|
if err == nil {
|
|
log.Println("wrote threshold config!")
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
return errors.New("admin tenant not found")
|
|
}
|
|
|
|
func (app *BackendApp) ConfigureAdminTenantOnThresholdServer() error {
|
|
|
|
log.Println("configuring threshold server...")
|
|
|
|
billingYear, billingMonth, _, _, _ := getBillingTimeInfo()
|
|
|
|
tenants, err := app.Model.GetTenants()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rows, err := app.Model.GetTenantVPSInstanceRows(billingYear, billingMonth)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
vpsInstances, err := app.Model.GetVPSInstances()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
actions := []func() taskResult{}
|
|
for _, row := range rows {
|
|
if row.TenantId == app.AdminTenantId {
|
|
vpsInstance, hasVpsInstance := vpsInstances[row.GetVPSInstanceId()]
|
|
tenant, hasTenant := tenants[row.TenantId]
|
|
if hasVpsInstance && hasTenant {
|
|
actions = append(actions, func() taskResult {
|
|
|
|
url := fmt.Sprintf("https://%s:%d/tunnels", vpsInstance.IPV4, app.ThresholdPort)
|
|
|
|
jsonBytes, err := json.Marshal([]ThresholdTunnel{
|
|
ThresholdTunnel{
|
|
ClientId: fmt.Sprintf("%d.%s", app.AdminTenantId, app.AdminThresholdNodeId),
|
|
ListenPort: 80,
|
|
ListenAddress: "0.0.0.0",
|
|
ListenHostnameGlob: fmt.Sprintf("%s.%s", tenant.Subdomain, app.FreeSubdomainDomain),
|
|
BackEndService: app.GreenhouseThresholdServiceId,
|
|
},
|
|
ThresholdTunnel{
|
|
ClientId: fmt.Sprintf("%d.%s", app.AdminTenantId, app.AdminThresholdNodeId),
|
|
ListenPort: 443,
|
|
ListenAddress: "0.0.0.0",
|
|
ListenHostnameGlob: fmt.Sprintf("%s.%s", tenant.Subdomain, app.FreeSubdomainDomain),
|
|
BackEndService: app.GreenhouseThresholdServiceId,
|
|
},
|
|
})
|
|
log.Println(string(jsonBytes))
|
|
if err != nil {
|
|
return taskResult{
|
|
Name: row.GetVPSInstanceId(),
|
|
Err: errors.Wrapf(
|
|
err,
|
|
"json serialization error calling %s (threshold admin tenant management API)",
|
|
url,
|
|
),
|
|
}
|
|
}
|
|
|
|
_, err = app.ThresholdProvisioning.MyHTTP200(
|
|
"PUT",
|
|
url,
|
|
bytes.NewBuffer(jsonBytes),
|
|
func(request *http.Request) { request.Header.Set("Content-Type", "application/json") },
|
|
)
|
|
if err != nil {
|
|
return taskResult{Name: row.GetVPSInstanceId(), Err: err}
|
|
}
|
|
return taskResult{Name: row.GetVPSInstanceId(), Err: nil}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(actions) == 0 {
|
|
return errors.New("admin tenant threshold server assignment not found")
|
|
}
|
|
|
|
errorStrings := []string{}
|
|
results := doInParallel(false, actions...)
|
|
for _, result := range results {
|
|
if result.Err != nil {
|
|
errorStrings = append(errorStrings, fmt.Sprintf("Can't update threshold tunnels for %s: %+v", result.Name, result.Err))
|
|
}
|
|
}
|
|
if len(errorStrings) > 0 {
|
|
return errors.Errorf("ConfigureThresholdServer(): \n%s", strings.Join(errorStrings, "\n"))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getInstanceProjectedUsage(
|
|
instance *VPSInstance,
|
|
workingAllocations *map[string]map[int]bool,
|
|
tenants *map[int]*TenantInfo,
|
|
desiredInstancesPerTenant int,
|
|
amountOfMonthElapsed float64,
|
|
) int64 {
|
|
|
|
usageByActiveTenantsSoFar := int64(0)
|
|
instanceAllocations, has := (*workingAllocations)[instance.GetId()]
|
|
if has {
|
|
for tenantId := range instanceAllocations {
|
|
if !(*tenants)[tenantId].Deactivated {
|
|
usageByActiveTenantsSoFar += (*tenants)[tenantId].Bytes
|
|
}
|
|
}
|
|
}
|
|
soFarFloat := float64(instance.Bytes)
|
|
activeTenantsSoFarFloat := float64(usageByActiveTenantsSoFar) / float64(desiredInstancesPerTenant)
|
|
totalProjectedUsageFloat := soFarFloat + ((activeTenantsSoFarFloat / amountOfMonthElapsed) - activeTenantsSoFarFloat)
|
|
return int64(totalProjectedUsageFloat)
|
|
}
|
|
|
|
func (app *BackendApp) SpawnNewMultitenantInstance() (*VPSInstance, error) {
|
|
|
|
userData := fmt.Sprintf(
|
|
`#!/bin/sh
|
|
touch /root/.hushlogin
|
|
%s
|
|
%s`,
|
|
app.DigitalOcean.GetSSHHostKeysFileScript(), app.BackblazeB2.GetFileUploadShellScript(),
|
|
)
|
|
|
|
log.Println("Creating new multitenant worker...")
|
|
instance, err := app.DigitalOcean.Create("multitenant-worker", userData)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to DigitalOcean.Create")
|
|
}
|
|
|
|
log.Printf("Waiting for %s to get an IPv4...\n", instance.GetId())
|
|
waitedSeconds := 0
|
|
for instance.IPV4 == "" && waitedSeconds < 300 {
|
|
time.Sleep(time.Second * time.Duration(5))
|
|
waitedSeconds += 5
|
|
instance, err = app.DigitalOcean.Get(instance.ProviderInstanceId)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to DigitalOcean.Get")
|
|
}
|
|
}
|
|
|
|
log.Printf("Waiting for %s (%s) to upload its ssh host public keys to backblaze...\n", instance.GetId(), instance.IPV4)
|
|
knownHostsFileContent, err := app.pollForSSHHostKeys(instance.GetId())
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to PollBackblazeForSSHHostKeys")
|
|
}
|
|
|
|
log.Printf("adding %s (%s) to the user's known-hosts file...\n", instance.GetId(), instance.IPV4)
|
|
err = app.SSH.AppendToKnownHostsFile(knownHostsFileContent)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to AppendToSSHKnownHostsFile")
|
|
}
|
|
|
|
log.Printf("Generating threshold install script for %s (%s)...\n", instance.GetId(), instance.IPV4)
|
|
ipv4 := net.ParseIP(instance.IPV4)
|
|
ipv6 := net.ParseIP(instance.IPV6)
|
|
ips := []net.IP{}
|
|
if ipv4 != nil {
|
|
ips = append(ips, ipv4)
|
|
}
|
|
if ipv6 != nil {
|
|
ips = append(ips, ipv6)
|
|
}
|
|
if len(ips) == 0 {
|
|
return nil, errors.Errorf("failed to convert instance IPs '%s' or '%s' to net.IP", instance.IPV4, instance.IPV6)
|
|
}
|
|
provisionThresholdScript, err := app.ThresholdProvisioning.GetServerInstallScript(instance.GetId(), ips)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to ThresholdProvisioning.GetServerInstallScript")
|
|
}
|
|
|
|
// TODO use potentially different username depending on cloud provider
|
|
log.Printf("Installing threshold on %s (%s)...\n", instance.GetId(), instance.IPV4)
|
|
stdout, stderr, err := app.SSH.RunScriptOnRemoteHost(provisionThresholdScript, "root", instance.IPV4)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(
|
|
err, "failed to RunScriptOnRemoteHost(provisionThresholdScript, \"root\", %s)",
|
|
instance.IPV4,
|
|
)
|
|
}
|
|
log.Println("stderr:")
|
|
log.Println(stderr)
|
|
log.Println("")
|
|
log.Println("")
|
|
log.Println("stdout:")
|
|
log.Println(stdout)
|
|
log.Println("")
|
|
log.Println("")
|
|
|
|
log.Printf("Saving %s (%s) info to the database...\n", instance.GetId(), instance.IPV4)
|
|
err = app.Model.CreateVPSInstance(instance)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to Model.CreateVPSInstance")
|
|
}
|
|
|
|
log.Printf("DONE! %s (%s) is provisioned!\n\n\n", instance.GetId(), instance.IPV4)
|
|
|
|
return instance, nil
|
|
}
|
|
|
|
func (app *BackendApp) pollForSSHHostKeys(instanceId string) (string, error) {
|
|
|
|
// it usually takes at least 15 seconds for an instance to boot and run
|
|
time.Sleep(time.Second * time.Duration(15))
|
|
|
|
iterations := 0
|
|
for iterations < 60 {
|
|
iterations++
|
|
fileContent, err := app.BackblazeB2.Get(fmt.Sprintf("greenhouse/known-hosts/%s", instanceId))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if fileContent != nil {
|
|
return string(fileContent), nil
|
|
}
|
|
|
|
time.Sleep(time.Second * time.Duration(5))
|
|
}
|
|
|
|
return "", fmt.Errorf("timed out waiting for %s to upload its host ssh public keys", instanceId)
|
|
}
|
|
|
|
func (app *BackendApp) saveVpsInstanceTenantSettings(
|
|
billingYear int,
|
|
billingMonth int,
|
|
instance *VPSInstance,
|
|
config map[int]*TunnelSettings,
|
|
) error {
|
|
|
|
url := fmt.Sprintf("https://%s:%d/tenants", instance.IPV4, app.ThresholdManagementPort)
|
|
|
|
jsonBytes, err := json.Marshal(config)
|
|
log.Println(string(jsonBytes))
|
|
if err != nil {
|
|
return errors.Wrapf(
|
|
err,
|
|
"json serialization error calling %s (threshold multitenant management API)",
|
|
url,
|
|
)
|
|
}
|
|
|
|
returnedBytes, err := app.MyHTTP200(
|
|
"PUT",
|
|
url,
|
|
bytes.NewBuffer(jsonBytes),
|
|
func(request *http.Request) { request.Header.Set("Content-Type", "application/json") },
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("returnedBytes: \n%s\n", string(returnedBytes))
|
|
|
|
returnedBytes2, err := app.MyHTTP200("GET", url, nil, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Printf("returnedBytes2: \n%s\n", string(returnedBytes2))
|
|
|
|
err = app.Model.SaveInstanceConfiguration(billingYear, billingMonth, instance, config)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "Can't save instance configuration to database for %s", instance.GetId())
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// TODO this fn needs to use app.MyHTTP() too
|
|
// func (app *BackendApp) getTenantSettingsFromVpsInstance(url string) (map[string]TunnelSettings, error) {
|
|
|
|
// httpClient := http.Client{
|
|
// Transport: &http.Transport{
|
|
// TLSClientConfig: app.ThresholdManagementTlsClientConfig,
|
|
// },
|
|
// Timeout: 10 * time.Second,
|
|
// }
|
|
// request, err := http.NewRequest("GET", url, nil)
|
|
// response, err := httpClient.Do(request)
|
|
// if err != nil {
|
|
// return nil, err
|
|
// }
|
|
// if response.StatusCode != 200 {
|
|
// return nil, fmt.Errorf("HTTP %d when calling %s (threshold multitenant management API)", response.StatusCode, url)
|
|
// }
|
|
// bytes, err := ioutil.ReadAll(response.Body)
|
|
// if err != nil {
|
|
// return nil, errors.Wrapf(err, "HTTP read error when calling %s (threshold multitenant management API)", url)
|
|
// }
|
|
// var responseObject map[string]TunnelSettings
|
|
// err = json.Unmarshal(bytes, &responseObject)
|
|
// if err != nil {
|
|
// return nil, errors.Wrapf(err, "JSON parse error when calling %s (threshold multitenant management API)", url)
|
|
// }
|
|
// return responseObject, nil
|
|
// }
|
|
|
|
func (service *BaseHTTPService) MyHTTP(
|
|
method string,
|
|
url string,
|
|
body io.Reader,
|
|
withRequest func(*http.Request),
|
|
) (int, []byte, error) {
|
|
|
|
if service.client == nil || service.clientExpiry != nil && time.Now().After(*(service.clientExpiry)) {
|
|
httpClient, httpClientExpiry, err := service.ClientFactory()
|
|
if err != nil {
|
|
return 0, nil, errors.Wrapf(err, "failed to create HTTP client when calling %s %s", method, url)
|
|
}
|
|
service.client = httpClient
|
|
service.clientExpiry = httpClientExpiry
|
|
}
|
|
|
|
request, err := http.NewRequest(method, url, body)
|
|
if err != nil {
|
|
return 0, nil, errors.Wrapf(err, "failed to create HTTP request calling %s %s", method, url)
|
|
}
|
|
if withRequest != nil {
|
|
withRequest(request)
|
|
}
|
|
response, err := service.client.Do(request)
|
|
if err != nil {
|
|
return 0, nil, errors.Wrapf(err, "HTTP request error when calling %s %s", method, url)
|
|
}
|
|
bytes, err := ioutil.ReadAll(response.Body)
|
|
if err != nil {
|
|
return response.StatusCode, nil, errors.Wrapf(err, "HTTP read error when calling %s %s ", method, url)
|
|
}
|
|
|
|
return response.StatusCode, bytes, err
|
|
}
|
|
|
|
func (service *BaseHTTPService) MyHTTP200(
|
|
method string,
|
|
url string,
|
|
body io.Reader,
|
|
withRequest func(*http.Request),
|
|
) ([]byte, error) {
|
|
|
|
statusCode, bytes, err := service.MyHTTP(method, url, body, withRequest)
|
|
if err == nil && statusCode >= 300 {
|
|
err = errors.Errorf("HTTP %d when calling %s %s: \n%s\n\n", statusCode, method, url, string(bytes))
|
|
}
|
|
|
|
return bytes, err
|
|
}
|
|
|
|
func doInParallel(stopOnFirstError bool, actions ...func() taskResult) map[string]taskResult {
|
|
|
|
resultsChannel := make(chan taskResult)
|
|
results := map[string]taskResult{}
|
|
|
|
//log.Printf("do %d actions in parallel", len(actions))
|
|
|
|
for _, action := range actions {
|
|
// this is how you do closures over the VALUE of a variable (not the variable itself) in golang
|
|
// https://stackoverflow.com/questions/26692844/captured-closure-for-loop-variable-in-go
|
|
action := action
|
|
go (func() {
|
|
result := action()
|
|
resultsChannel <- result
|
|
})()
|
|
}
|
|
|
|
for range actions {
|
|
result := <-resultsChannel
|
|
results[result.Name] = result
|
|
if result.Err != nil && stopOnFirstError {
|
|
break
|
|
}
|
|
}
|
|
|
|
return results
|
|
}
|
|
|
|
func int64Abs(x int64) int64 {
|
|
if x > 0 {
|
|
return x
|
|
} else {
|
|
return -x
|
|
}
|
|
}
|
|
|
|
func getRandomPermutation(length int) []int {
|
|
toReturn := make([]int, length)
|
|
toAppend := make([]int, length)
|
|
|
|
for i := 0; i < length; i++ {
|
|
toAppend[i] = i
|
|
}
|
|
|
|
for i := 0; i < length; i++ {
|
|
picked := mathRand.Intn(length - i)
|
|
toReturn[i] = toAppend[picked]
|
|
toAppend[picked] = toAppend[length-i]
|
|
toAppend = toAppend[0 : length-i]
|
|
}
|
|
|
|
return toReturn
|
|
}
|
|
|
|
func getKnapsackSolutionScore(scale int64, proposedDistance int64, proposedLength int) int64 {
|
|
return proposedDistance + int64(float64(scale)*knapsackShufflingMinimizationFactor)*int64(proposedLength)
|
|
}
|