Use cloud logging (#254)

Use "cloud.google.com/go/logging" where possible, and remove most
instances of printing to os.Stdout or os.Stderr, replacing them with
calls to logging functions.

api/api.go

@@ -3,8 +3,8 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
-	"os"
 	"strings"
 	"time"
 
@@ -17,6 +17,7 @@ type API struct {
 	hstspreload hstspreloadWrapper
 	preloadlist preloadlistWrapper
 	cache       *cache
+	logger      *log.Logger
 }
 
 const (
@@ -25,12 +26,13 @@ const (
 
 // New creates a new API struct with the given database and the proper
 // unexported fields.
-func New(db database.Database) API {
+func New(db database.Database, logger *log.Logger) API {
 	return API{
 		database:    db,
 		hstspreload: actualHstspreload{},
 		preloadlist: actualPreloadlist{},
 		cache:       cacheWithDuration(defaultCacheDuration),
+		logger:      logger,
 	}
 }
 
@@ -54,7 +56,7 @@ func (api API) CheckConnection() error {
 	_, err := api.database.StateForDomain("garron.net")
 	if err != nil {
 		if strings.Contains(err.Error(), "missing project/dataset id") {
-			fmt.Fprintf(os.Stderr, "Try running: make serve\n")
+			api.logger.Print("Try running: make serve")
 		}
 		return err
 	}

api/api_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"bytes"
 	"encoding/json"
+	"log"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -60,6 +61,7 @@ func mockAPI(cacheDuration time.Duration) (api API, mc *database.MockController,
 		hstspreload: h,
 		preloadlist: c,
 		cache:       cacheWithDuration(cacheDuration),
+		logger:      log.Default(),
 	}
 	return api, mc, h, c
 }

api/domain_handlers.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"strings"
 	"sync"
@@ -429,23 +428,23 @@ func (api API) RemoveIneligibleDomains(w http.ResponseWriter, r *http.Request) {
 	// ineligible domain database
 	var deleteEligibleDomains []string
 
-	log.Print("Fetching domains...")
+	api.logger.Print("Fetching domains...")
 	// Get all domains
 	domains, err := api.database.AllDomainStates()
 	if err != nil {
 		msg := fmt.Sprintf("Internal error: could not retrieve domains. (%s)\n", err)
 		http.Error(w, msg, http.StatusInternalServerError)
 		return
 	}
-	log.Print("Filtering domains...")
+	api.logger.Print("Filtering domains...")
 
 	// Filter Domains
 	for _, d := range domains {
 		if d.Policy == preloadlist.Bulk18Weeks || d.Policy == preloadlist.Bulk1Year {
 			policyStates[d.Name] = d
 		}
 	}
-	log.Print("Getting ineligible domain states...")
+	api.logger.Print("Getting ineligible domain states...")
 
 	// call GetIneligibleDomainStates, add to map
 	states := make(map[string]database.IneligibleDomainState)
@@ -465,7 +464,7 @@ func (api API) RemoveIneligibleDomains(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	log.Printf("Starting to scan %d domains\n", len(policyStates))
+	api.logger.Printf("Starting to scan %d domains\n", len(policyStates))
 	statesAndIssues := make(chan DomainStateWithIssues)
 	domainStates := make(chan database.DomainState)
 	var wg sync.WaitGroup
@@ -482,20 +481,20 @@ func (api API) RemoveIneligibleDomains(w http.ResponseWriter, r *http.Request) {
 			}
 		}()
 	}
-	log.Printf("Started %d workers\n", numWorkers)
+	api.logger.Printf("Started %d workers\n", numWorkers)
 	go func() {
 		i := 0
 		for _, d := range policyStates {
 			i++
 			wg.Add(1)
 			domainStates <- d
 			if i % 1000 == 0 {
-				log.Printf("Sent %d domains to workers\n", i)
+				api.logger.Printf("Sent %d domains to workers\n", i)
 			}
 		}
 
 		wg.Wait()
-		log.Println("... all goroutines done, closing channels")
+		api.logger.Println("... all goroutines done, closing channels")
 		close(domainStates)
 		close(statesAndIssues)
 	}()
@@ -535,11 +534,11 @@ func (api API) RemoveIneligibleDomains(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	log.Printf("About to set %d domains as potentially ineligible\n", len(ineligibleDomains))
+	api.logger.Printf("About to set %d domains as potentially ineligible\n", len(ineligibleDomains))
 
 	// Add ineligible domains to the database
 	err = api.database.SetIneligibleDomainStates(ineligibleDomains, func(format string, args ...interface{}) {
-		log.Printf(format, args...)
+		api.logger.Printf(format, args...)
 	})
 	if err != nil {
 		msg := fmt.Sprintf("Internal error: could not set domains. (%s)\n", err)

database/database.go

@@ -11,7 +11,6 @@ import (
 
 const (
 	localProjectID = "hstspreload-local"
-	prodProjectID  = "hstspreload"
 
 	batchSize = 450
 	timeout   = 90 * time.Second
@@ -50,8 +49,8 @@ func TempLocalDatabase() (db DatastoreBacked, shutdown func() error, err error)
 
 // ProdDatabase gives a Database that will call out to
 // the real production instance of Google Cloud Datastore
-func ProdDatabase() (db DatastoreBacked) {
-	return DatastoreBacked{gcd.NewProdBackend(), prodProjectID}
+func ProdDatabase(projectID string) DatastoreBacked {
+	return DatastoreBacked{gcd.NewProdBackend(), projectID}
 }
 
 var blackholeLogf = func(format string, args ...interface{}) {}

database/database_test.go

@@ -2,6 +2,7 @@ package database
 
 import (
 	"fmt"
+	"log"
 	"os"
 	"reflect"
 	"sort"
@@ -29,8 +30,7 @@ func ExampleTempLocalDatabase() {
 func TestMain(m *testing.M) {
 	localDatabase, shutdown, err := TempLocalDatabase()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "could not initialize local backend: %s", err)
-		os.Exit(1)
+		log.Fatalf("could not initialize local backend: %s", err)
 	}
 
 	testDB = localDatabase

go.mod

@@ -3,17 +3,11 @@ module github.com/chromium/hstspreload.org
 go 1.15
 
 require (
-	cloud.google.com/go/datastore v1.12.1
+	cloud.google.com/go/datastore v1.15.0
 	github.com/chromium/hstspreload v0.0.0-20230717215815-2404b4b967f9
 	golang.org/x/net v0.23.0
-	google.golang.org/api v0.132.0
-	google.golang.org/grpc v1.56.3
+	google.golang.org/api v0.149.0
+	google.golang.org/grpc v1.59.0
 )
 
-require (
-	cloud.google.com/go v0.110.6 // indirect
-	cloud.google.com/go/compute v1.22.0 // indirect
-	google.golang.org/genproto v0.0.0-20230720185612-659f7aaaa771 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230720185612-659f7aaaa771 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect
-)
+require cloud.google.com/go/logging v1.9.0