add: logger, shutdown API

go.mod

@@ -10,7 +10,8 @@ require (
 	github.com/huichen/murmur v0.0.0-20130808212358-e0489551cf51
 	github.com/huichen/sego v0.0.0-20180617034105-3f3c8a8cfacc
 	github.com/issue9/assert v1.4.1 // indirect
-	github.com/rs/zerolog v1.23.0 // indirect
+	github.com/rs/zerolog v1.23.0
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	modernc.org/kv v1.0.3
 )

go.sum

@@ -50,6 +50,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 modernc.org/bufs v1.0.0/go.mod h1:0FCJ1mAhGiiFos8v2PISFhP2fEj3CK4/ACCLidqU/Ok=
 modernc.org/exp v1.0.0/go.mod h1:nfZshN9GU3d+mveZaR6VW2wJfh8gVBDgp8i2NZ/pFM8=
 modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w=

wukongd/config.go

@@ -18,6 +18,7 @@ type server struct {
 
 type Loggerger struct {
 	File, Level string
+	Console     bool
 }
 
 type engine struct {

wukongd/http.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/huichen/wukong/types"
 	"net/http"
+	"os"
 	"strconv"
 	"time"
 )
@@ -43,7 +44,9 @@ func startHttp(addr string) error {
 	v1.HandleFunc("/delete_index", delete_index_v1).Methods("POST")
 	v1.HandleFunc("/search", search_v1).Methods("GET").Queries("keywords", "{keywords}",
 		"offset", "{offset:[0-9]+}", "limit", "{offset:[0-9]+}")
-	v1.HandleFunc("/shutdown", shutdown).Methods("POST")
+	v1.HandleFunc("/shutdown", shutdown_v1).Methods("POST")
+	v1.HandleFunc("/stat", stat_v1).Methods("GET")
+	v1.HandleFunc("/index/{docId}", index_get_v1).Methods("GET")
 	router.Use(mux.CORSMethodMiddleware(router))
 
 	_httpServer = &http.Server{
@@ -55,7 +58,7 @@ func startHttp(addr string) error {
 
 	go func() {
 		if err := _httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			fmt.Println("http server error: ", err)
+			logger.Error().Err(err).Msg("http server error")
 		}
 	}()
 
@@ -82,7 +85,8 @@ func index_v1(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	fmt.Println("indexing:", param)
+	logger.Trace().Interface("param", param).Msg("new index")
+
 	searcher.IndexDocumentS(param.DocId, types.DocumentIndexData{
 		Content: param.Content,
 	}, param.ForceUpdate)
@@ -91,6 +95,10 @@ func index_v1(w http.ResponseWriter, r *http.Request) {
 	_, err = w.Write([]byte(`{"ec":0,"result":{}}`))
 }
 
+func index_get_v1(w http.ResponseWriter, r *http.Request) {
+
+}
+
 // delete index
 func delete_index_v1(w http.ResponseWriter, r *http.Request) {
 	var param ParamDelete
@@ -99,7 +107,7 @@ func delete_index_v1(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	fmt.Println("removing index:", param)
+	logger.Trace().Interface("param", param).Msg("delete index")
 
 	searcher.RemoveDocumentS(param.DocId, param.ForceUpdate)
 
@@ -113,7 +121,9 @@ func search_v1(w http.ResponseWriter, r *http.Request) {
 	keywords := params["keywords"]
 	offset, _ := strconv.ParseInt(params["offset"], 10, 64)
 	limit, _ := strconv.ParseInt(params["limit"], 10, 64)
-	fmt.Println("searching:", params)
+
+	logger.Trace().Interface("param", params).Msg("search")
+
 	resp := searcher.Search(types.SearchRequest{
 		Text: keywords,
 		RankOptions: &types.RankOptions{
@@ -130,6 +140,11 @@ func search_v1(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprintf(w, `{"ec":0,"result":%s}`, string(sresp))
 }
 
-func shutdown(w http.ResponseWriter, r *http.Request) {
+func shutdown_v1(w http.ResponseWriter, r *http.Request) {
+	logger.Trace().Msg("shutdown")
+	chSig <- os.Interrupt
+}
 
+func stat_v1(w http.ResponseWriter, r *http.Request) {
+	logger.Trace().Msg("stat")
 }

wukongd/logger.go

@@ -1 +1,50 @@
 package main
+
+import (
+	"fmt"
+	"github.com/rs/zerolog"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"io"
+	"os"
+	"path"
+	"strings"
+)
+
+var (
+	logger zerolog.Logger
+
+	levelMap = map[string]zerolog.Level{
+		"trace":   zerolog.TraceLevel,
+		"debug":   zerolog.DebugLevel,
+		"info":    zerolog.InfoLevel,
+		"warn":    zerolog.WarnLevel,
+		"error":   zerolog.ErrorLevel,
+		"disable": zerolog.NoLevel,
+	}
+)
+
+func initLogger(console bool, file string, level string) {
+	if err := os.MkdirAll(path.Dir(file), 744); err != nil {
+		fmt.Println("bad")
+	}
+	fileWriter := &lumberjack.Logger{
+		Filename:   file,
+		MaxBackups: 128,
+		MaxSize:    10,
+		MaxAge:     30,
+	}
+	var writers = []io.Writer{fileWriter}
+	if console {
+		writers = append(writers, zerolog.ConsoleWriter{Out: os.Stderr})
+	}
+	mw := zerolog.MultiLevelWriter(writers...)
+	newlogger := zerolog.New(mw).With().Timestamp().Caller().Logger()
+
+	lv, ok := levelMap[strings.ToLower(level)]
+	if !ok {
+		lv = zerolog.InfoLevel
+	}
+	zerolog.SetGlobalLevel(lv)
+
+	logger = newlogger
+}

wukongd/main.go

@@ -3,18 +3,22 @@ package main
 import (
 	"flag"
 	"fmt"
-	wukong "github.com/huichen/wukong/engine"
 	"github.com/huichen/wukong/types"
 	"os"
 	"os/signal"
+	"syscall"
+)
+
+import (
+	wukong "github.com/huichen/wukong/engine"
 )
 
 /**
-  wukongd 基于 wukong 实现，内置一个 http server. 和一个启动配置文件
+wukongd 基于 wukong 实现，内置一个 http server. 和一个启动配置文件
 
-  * 支持字符串索引
+* 支持字符串索引
 
-  多实例部署可直接启动多个实例，http 客户端来实现索引的 hash-sharding 以及 查询汇总
+多实例部署可直接启动多个实例，http 客户端来实现索引的 hash-sharding 以及 查询汇总
 
 */
 
@@ -23,9 +27,11 @@ var (
 	confile     = flag.String("c", "wukongd.conf", "specify the config file")
 	test_conf   = flag.String("t", "", "test specified config file")
 	showVersion = flag.Bool("v", false, "display version")
+
+	chSig = make(chan os.Signal, 2)
 )
 
-func main() {
+func _main() {
 	flag.Parse()
 
 	var err error
@@ -34,7 +40,6 @@ func main() {
 		printVersion()
 		return
 	}
-	fmt.Println(*test_conf)
 	// check config
 	if len(*test_conf) > 0 {
 		_, err := loadConfig(*test_conf)
@@ -55,6 +60,8 @@ func main() {
 	printBaseInfo()
 
 	// init logger
+	initLogger(config.Logger.Console, config.Logger.File, config.Logger.Level)
+
 	// init engine
 	options := types.RankOptions{
 		OutputOffset: config.Engine.OutputOffset,
@@ -69,24 +76,28 @@ func main() {
 		NumShards:               config.Engine.ShardsNum,
 		DefaultRankOptions:      &options,
 		UsePersistentStorage:    config.Engine.Persistent,
+		PersistentStorageEngine: config.Engine.PersistentEngine,
 		PersistentStorageFolder: config.Engine.PersistentFolder,
 		PersistentStorageShards: config.Engine.PersistentShardsNum,
 	})
 	searcher.FlushIndex()
-	fmt.Println("indexed:", searcher.NumDocumentsIndexed())
+	logger.Info().Uint64("indexed", searcher.NumDocumentsIndexed()).Msg("index finished")
 	// init http
 	err = startHttp(config.Server.Addr)
 	if err != nil {
 		panic(err)
 	}
+	logger.Info().Str("addr", config.Server.Addr).Msg("server started")
+
+	signal.Notify(chSig, os.Interrupt, syscall.SIGTERM, syscall.SIGINT)
+
+	sig := <-chSig
+	logger.Info().Str("signal", sig.String()).Msg("received signal, stopping")
 
-	fmt.Println("server started.")
-	ch := make(chan os.Signal, 1)
-	signal.Notify(ch, os.Interrupt)
-	fmt.Println("received signal: ", <-ch, " stopping...")
 	stopHttp()
 	searcher.Close()
-	fmt.Println("stopped.")
+
+	logger.Info().Msg("server stopped")
 }
 
 func printBaseInfo() {
@@ -102,3 +113,7 @@ log level: %s
 		config.Logger.File,
 		config.Logger.Level)
 }
+
+func main() {
+	_main()
+}

wukongd/wukongd.conf.sample

@@ -5,6 +5,7 @@ mode = 'debug'
 [logger]
 file  ='/var/log/wukongd.log'
 level = 'debug'
+console = true
 
 [engine]
 dictionary = 'd:/projects/golang/wukongd/data/dictionary.txt'