Merge pull request #44 from covalenthq/logfolder

add param to bspagent cmd specifying folder for file logging

Author: noslav

README.md

@@ -35,18 +35,18 @@
 
 # BSP Agent
 
-* [Introduction](#agent_intro)
-  * [Resources](#agent_resources)
-* [Architecture](#agent_arch)
-* [Block-Replica](#agent_block)
-  * [State-Specimen](#state_specimen)
-* [Environment](#environment)
-* [Build & Run](#build_run)
-  * [Flag Definitions](#flag_definitions)
-* [Docker](#docker)
-* [Scripts](#scripts)
-  * [Inspect](#inspect)
-* [Contributing](./docs/CONTRIBUTING.md)
+- [BSP Agent](#bsp-agent)
+  - [<span id="agent_intro">Introduction</span>](#introduction)
+  - [<span id="agent_resources">Resources</span>](#resources)
+  - [<span id="agent_arch">Architecture</span>](#architecture)
+  - [<span id="agent_block">Block-replica</span>](#block-replica)
+    - [<span id="state_specimen">State-specimen</span>](#state-specimen)
+  - [<span id="environment">Environment</span>](#environment)
+  - [<span id="build_run">Build & Run</span>](#build--run)
+    - [<span id="flag_definitions">Flag definitions</span>](#flag-definitions)
+  - [<span id="docker">Docker</span>](#docker)
+  - [<span id="scripts">Scripts</span>](#scripts)
+    - [<span id="inspect">Inspect</span>](#inspect)
 
 ## <span id="agent_intro">Introduction</span>
 
@@ -206,7 +206,8 @@ go run ./cmd/bspagent/*.go \
     --replica-bucket="<covalenthq-geth-block-replica-bucket>" \
     --segment-length=1 \
     --proof-chain-address="0xe9048412727c96f1044c78CffA45BB2311aE1F1D" \
-    --consumer-timeout=80
+    --consumer-timeout=80 \
+    --logs-folder ./logs/
 ```
 
 Or update the Makefile with the correct --gcp-svc-account, --replica-bucket & --proof-chain-address and run with the following.
@@ -217,7 +218,7 @@ Or update the Makefile with the correct --gcp-svc-account, --replica-bucket & --
 
 ### <span id="flag_definitions">Flag definitions</span>
 
---redis-url - this flag tells the BSP agent where to find the BSP messages, the stream topic key `replication` and the consumer group name with the field after "#" that in this case is `replicate`, additionally one can provide a password to the redis instance here but we recommend that by adding the line below to the .envrc
+`--redis-url` - this flag tells the BSP agent where to find the BSP messages, the stream topic key `replication` and the consumer group name with the field after "#" that in this case is `replicate`, additionally one can provide a password to the redis instance here but we recommend that by adding the line below to the .envrc
 
 ```env
 export REDIS_PWD=your-redis-pwd
@@ -237,6 +238,8 @@ export REDIS_PWD=your-redis-pwd
 
 `--consumer-timeout` - specifies in how many seconds the BSP agent stops waiting for new messages from the redis pending queue for decode, pack, encode, proof, store and upload.
 
+`--logs-folder` - specifies the location (folder) where the log files have to be placed. In case of error (like permission errors), the logs are not recorded in files.
+
 ## <span id="docker">Docker</span>
 
 Please install [docker and docker-compose](https://docs.docker.com/compose/install/).

cmd/bspagent/main.go

@@ -5,8 +5,11 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"os/signal"
+	"path"
 	"strings"
 	"sync"
 	"syscall"
@@ -21,6 +24,7 @@ import (
 	uuid "github.com/satori/go.uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/ubiq/go-ubiq/rlp"
+	"golang.org/x/sys/unix"
 	"gopkg.in/avro.v0"
 	"gopkg.in/natefinch/lumberjack.v2"
 
@@ -49,6 +53,7 @@ var (
 	proofChainFlag             string
 	binaryFilePathFlag         string
 	websocketURLsFlag          string
+	logsFolderFlag             = "./logs/"
 
 	// stream processing vars
 	start                 = ">"
@@ -60,25 +65,7 @@ var (
 	blockReplica          types.BlockReplica
 )
 
-func init() {
-	formatter := runtime.Formatter{ChildFormatter: &log.TextFormatter{
-		FullTimestamp: true,
-	}}
-	formatter.Line = true
-	log.SetFormatter(&formatter)
-	bspLoggerOutput := utils.NewLoggerOut(os.Stdout, &lumberjack.Logger{
-		// logs folder created/searched in directory in which agent was started.
-		Filename:   "./logs/log.log",
-		MaxSize:    100, // megabytes
-		MaxBackups: 7,
-		MaxAge:     10, // days
-	})
-	log.SetOutput(&bspLoggerOutput)
-	log.SetLevel(log.InfoLevel)
-	log.WithFields(log.Fields{"file": "main.go"}).Info("bsp-agent is running...")
-}
-
-func main() {
+func parseFlags() {
 	flag.StringVar(&redisURLFlag, "redis-url", utils.LookupEnvOrString("RedisURL", redisURLFlag), "redis consumer stream url")
 	flag.StringVar(&avroCodecPathFlag, "avro-codec-path", utils.LookupEnvOrString("CodecPath", avroCodecPathFlag), "local path to AVRO .avsc files housing the specimen/result schemas")
 	flag.StringVar(&binaryFilePathFlag, "binary-file-path", utils.LookupEnvOrString("BinaryFilePath", binaryFilePathFlag), "local path to AVRO encoded binary files that contain block-replicas")
@@ -88,8 +75,43 @@ func main() {
 	flag.StringVar(&websocketURLsFlag, "websocket-urls", utils.LookupEnvOrString("WebsocketURLs", websocketURLsFlag), "url to websockets clients separated by space")
 	flag.IntVar(&segmentLengthFlag, "segment-length", utils.LookupEnvOrInt("SegmentLength", segmentLengthFlag), "number of block specimen/results within a single uploaded avro encoded object")
 	flag.IntVar(&consumerPendingTimeoutFlag, "consumer-timeout", utils.LookupEnvOrInt("ConsumerPendingTimeout", consumerPendingTimeoutFlag), "number of seconds to wait before pending messages consumer timeout")
+	flag.StringVar(&logsFolderFlag, "logs-folder", utils.LookupEnvOrString("LogsFolder", logsFolderFlag), "Location where the log files should be placed")
 	flag.Parse()
+}
+
+func init() {
+	parseFlags()
+
+	// setup logger
+	formatter := runtime.Formatter{ChildFormatter: &log.TextFormatter{
+		FullTimestamp: true,
+	}}
+	formatter.Line = true
+	log.SetFormatter(&formatter)
 
+	var outWriter io.Writer
+	logLocationURL, err := getLogLocationURL(logsFolderFlag)
+	if err != nil {
+		log.Warn("error while setting up file logging: ", err)
+		outWriter = os.Stdout
+	} else {
+		logFilePath := path.Join(logLocationURL.Path, "log.log")
+		bspLogger := utils.NewLoggerOut(os.Stdout, &lumberjack.Logger{
+			// logs folder created/searched in directory in which agent was started.
+			Filename:   logFilePath,
+			MaxSize:    100, // megabytes
+			MaxBackups: 7,
+			MaxAge:     10, // days
+		})
+		outWriter = &bspLogger
+	}
+
+	log.SetOutput(outWriter)
+	log.SetLevel(log.InfoLevel)
+	log.WithFields(log.Fields{"file": "main.go"}).Info("bsp-agent is running...")
+}
+
+func main() {
 	config, err := config.LoadConfig()
 	if err != nil {
 		panic(err)
@@ -284,3 +306,27 @@ func processStream(config *config.Config, replicaCodec *goavro.Codec, redisClien
 		}
 	}
 }
+
+func getLogLocationURL(logPath string) (*url.URL, error) {
+	logLocation := utils.ExpandPath(logPath)
+	locationURL, err := url.Parse(logLocation)
+	if err == nil {
+		if _, existErr := os.Stat(locationURL.Path); os.IsNotExist(existErr) {
+			// directory doesn't exist, create
+			createErr := os.Mkdir(locationURL.Path, os.ModePerm)
+			if createErr != nil {
+				return nil, fmt.Errorf("error creating the directory: %w", createErr)
+			}
+		}
+
+		if !writable(locationURL.Path) {
+			return nil, fmt.Errorf("write access not present for given log location")
+		}
+	}
+
+	return locationURL, fmt.Errorf("log-location: %w", err)
+}
+
+func writable(path string) bool {
+	return unix.Access(path, unix.W_OK) == nil
+}

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/ubiq/go-ubiq v3.0.1+incompatible
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158
 	google.golang.org/api v0.70.0
 	gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0

go.sum

@@ -724,6 +724,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220318055525-2edf467146b5 h1:saXMvIOKvRFwbOMicHXr0B1uwoxq9dGmLe5ExMES6c4=
+golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/utils/utils.go

@@ -10,6 +10,8 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"os/user"
+	"path"
 	"strconv"
 	"strings"
 
@@ -166,3 +168,30 @@ func DecodeAvro(record avro.AvroRecord, buffer []byte) error {
 
 	return reader.Read(record, decoder)
 }
+
+// ExpandPath expands a file path
+// 1. replace tilde with users home dir
+// 2. expands embedded environment variables
+// 3. cleans the path, e.g. /a/b/../c -> /a/c
+// Note, it has limitations, e.g. ~someuser/tmp will not be expanded
+func ExpandPath(fsPath string) string {
+	if strings.HasPrefix(fsPath, "~/") || strings.HasPrefix(fsPath, "~\\") {
+		if home := HomeDir(); home != "" {
+			fsPath = home + fsPath[1:]
+		}
+	}
+
+	return path.Clean(os.ExpandEnv(fsPath))
+}
+
+// HomeDir returns full path of home directory for current user
+func HomeDir() string {
+	if home := os.Getenv("HOME"); home != "" {
+		return home
+	}
+	if usr, err := user.Current(); err == nil {
+		return usr.HomeDir
+	}
+
+	return ""
+}