diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..77ecf11 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.11 + +COPY ./server/server_bin /go/src/ +COPY ./server /go/src/server +COPY ./client /go/src/client +RUN go get -u golang.org/x/lint/golint && \ +go get golang.org/x/tools/cmd/goimports && \ +go get github.com/golang/go/src/cmd/vet + +WORKDIR /go/src/ +ENTRYPOINT ["/go/src/server_bin"] +EXPOSE 9090 \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..512907f --- /dev/null +++ b/Makefile @@ -0,0 +1,84 @@ +################ Preparation #################### +REPOSITORY_PATH_SERVER := kv_storage_server +REPOSITORY_PATH_CLIENT := kv_storage_client +REGISTRY := agri88 +VERSION := $(shell cat VERSION) +BECOME := sudo -E +DOCKER_IMAGE := $(REGISTRY)/kv_storage:$(VERSION) +DOCKER_BUILDER := golang:1.11 +DOCKER_RUNNER_SERVER = docker run --rm -v $(CURDIR)/server/:/go/src/$(REPOSITORY_PATH_SERVER) +DOCKER_RUNNER_SERVER += $(DOCKER_ENVS) -w /go/src/$(REPOSITORY_PATH_SERVER) +DOCKER_RUNNER_CLIENT = docker run --rm -v $(CURDIR)/client/:/go/src/$(REPOSITORY_PATH_CLIENT) +DOCKER_RUNNER_CLIENT += $(DOCKER_ENVS) -w /go/src/$(REPOSITORY_PATH_CLIENT) + # search files for fmt and check targets, excluding "vendor" folder +SEARCH_GOFILES = find -not -path '*/vendor/*' -type f -name "*.go" +BUILDER_SERVER = $(DOCKER_RUNNER_SERVER) $(DOCKER_BUILDER) +BUILDER_CLIENT = $(DOCKER_RUNNER_CLIENT) $(DOCKER_BUILDER) +PORT = 9090 +CONTAINER_NAME = kv_server +IMAGE_ID := $(shell $(BECOME) docker images -q $(DOCKER_IMAGE)) + ################ End Preparation #################### + ################ Binary Target #################### +build: clean server_bin client_bin + +.PHONY: server_bin +server_bin: + mkdir ./server/common_files && \ + mkdir ./server/log && \ + $(BECOME) $(BUILDER_SERVER) go build -o $@ ./$(@D) + $(BECOME) docker build -t $(DOCKER_IMAGE) ./ +.PHONY: client_bin +client_bin: + $(BECOME) $(BUILDER_CLIENT) go build -o $@ ./$(@D) + ################ Clean Target #################### +.PHONY: clean +clean: + $(BECOME) $(RM) ./server/server_bin + $(BECOME) $(RM) ./client/client_bin + $(BECOME) $(RM) -r ./server/log + $(BECOME) $(RM) -r ./server/common_files +ifneq ($(shell $(BECOME) docker ps -q -f name=$(CONTAINER_NAME)), ) + $(BECOME) docker rm -f $(CONTAINER_NAME) +endif +ifneq ($(IMAGE_ID), ) + $(BECOME) docker image rm $(IMAGE_ID) +endif + ################ Format and Validate Targets #################### +.PHONY: check +check: + $(BECOME) docker run --name $(CONTAINER_NAME)_check -d $(DOCKER_IMAGE) + $(BECOME) docker exec -it $(CONTAINER_NAME)_check gofmt -s -l ./server + $(BECOME) docker exec -it $(CONTAINER_NAME)_check golint ./server + $(BECOME) docker exec -it $(CONTAINER_NAME)_check go vet ./server + $(BECOME) docker exec -it $(CONTAINER_NAME)_check gofmt -s -l ./client + $(BECOME) docker exec -it $(CONTAINER_NAME)_check golint ./client + $(BECOME) docker exec -it $(CONTAINER_NAME)_check go vet ./client + $(BECOME) docker rm -f $(CONTAINER_NAME)_check + ################ Docker Targets #################### +.PHONY: run +ifneq ($(shell $(BECOME) docker ps -a -q -f name= $(CONTAINER_ID)), ) +run: + echo container is created +else +ifneq ($(IMAGE_ID), ) +run: + $(BECOME) docker run --name $(CONTAINER_NAME) -p $(PORT):9090 -v $(CURDIR)/server/log/:/go/src/log/ -v $(CURDIR)/server/common_files/:/go/src/common_files/ -d $(DOCKER_IMAGE) +else +run: build + $(BECOME) docker run --name $(CONTAINER_NAME) -p $(PORT):9090 -v $(CURDIR)/server/log/:/go/src/log/ -v $(CURDIR)/server/common_files/:/go/src/common_files/ -d $(DOCKER_IMAGE) +endif + +endif + +.PHONY: stop +ifneq ($(shell $(BECOME) docker ps -a -q -f name= $(CONTAINER_ID)), ) +stop: + $(BECOME) docker rm -f $(CONTAINER_NAME) +else +stop: + echo container not exist +endif + ################ Test Targets #################### +.PHONY: test +test: + $(BECOME) docker exec -it $(CONTAINER_NAME) go test ./server/ -coverprofile ./common_files/cover.out diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..93aa114 --- /dev/null +++ b/client/client.go @@ -0,0 +1,54 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "net" + "os" +) + +func main() { + + var port string + flag.StringVar(&port, "port", "9090", "listening port") + flag.StringVar(&port, "p", "9090", "listening port") + var host string + flag.StringVar(&host, "h", "127.0.0.1", "listening IP") + flag.StringVar(&host, "host", "127.0.0.1", "listening IP") + + flag.Parse() + + address := fmt.Sprintf("%v:%v", host, port) + conn, err := net.Dial("tcp", address) + if err != nil { + fmt.Println(err) + return + } + defer conn.Close() + for { + var source string + fmt.Print("Enter command: ") + myscanner := bufio.NewScanner(os.Stdin) + myscanner.Scan() + source = myscanner.Text() + if len(source) == 0 { + fmt.Println("Wrong input: no command") + continue + } + // send message + if n, err := conn.Write([]byte(source)); n == 0 || err != nil { + fmt.Println(err) + return + } + // get response + fmt.Print("Server response:") + buff := make([]byte, 1024) + n, err := conn.Read(buff) + if err != nil { + break + } + fmt.Print(string(buff[0:n])) + fmt.Println() + } +} diff --git a/readme b/readme new file mode 100644 index 0000000..b82e46f --- /dev/null +++ b/readme @@ -0,0 +1,45 @@ +use makefile for all actions + +make build: + compiling server & client binaries use latest golang docker images, before do that, runs clean command + +make run: + runs docker container using earlier built image. Before, if image not exist, build command run automatically + ./server/common_files directory contains storage file. optional may contain cover.out file after tests run + ./server/log directory contains server log file + +make check: + runs golint, go fmt and go vet for verify code + +make test: + runs tests if package contains it. cover.out file put into common_files dir + +make stop: + force delete docker container if exist it + +make clean: + runs for full cleaning project. Delete docker container, docker image, binaries and created directories and + all created files + + +Work with application + +Server +You may run server using flags: + -p --port listening port, default is 9090; + -h --host ip address, default is 0.0.0.0 (listening all ip); + -m --mode storage mode, default is "memory", alternate mode is "disk" (save storage to disk) + + +Client +You may run client using flags: + -p --port connect to port, default is 9090; + -h --host connect to ip address, default is 127.0.0.1 + +Commands: + set key value + get key + del key + keys [searching key] + + diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..0f3945b --- /dev/null +++ b/server/server.go @@ -0,0 +1,284 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "io/ioutil" + "log" + "net" + "os" + "regexp" + "strings" +) + +func (object storage) GET(key string, response chan string) { + value, ok := object.storageMap[key] + if !ok && object.storageMode == "disk" { + valueInFile := object.readFromFile(key) + if valueInFile != "false" { + object.storageMap[key] = valueInFile + response <- key + ":" + valueInFile //send data to client + } else { + response <- "no pair contains key=" + key //send data to client + } + } else { + response <- key + ":" + value //send data to client + } +} + +func (object storage) SET(key string, value string, response chan string) { + val, ok := object.storageMap[key] + if ok == true { + response <- fmt.Sprintf("store contains pair with key %v: %v", key, val) //send data to client + } else { + inFileValue := object.readFromFile(key) + if inFileValue != "false" { + object.storageMap[key] = inFileValue + response <- "pair with key " + key + "exist. " + key + ": " + inFileValue //send data to client + } else { + object.storageMap[key] = value + response <- "pair " + key + ":" + value + " created" //send data to client + } + } +} + +func (object storage) DEL(key string, response chan string) { + _, ok := object.storageMap[key] + if ok == true { + delete(object.storageMap, key) + object.deleteFromFile(key) + response <- "pair deleted" //send data to client + } else { + deleteFromFileResult := object.deleteFromFile(key) + if deleteFromFileResult == true { + response <- "pair deleted" //send data to client + } else { + response <- "no pair for delete" //send data to client + } + } +} + +func (object storage) KEYS(pattern string, response chan string) { + result := make(map[string]string) + for key, value := range object.storageMap { + keyExist, err := regexp.MatchString(pattern, key) + if err != nil { + result["error"] = "true" + } else if keyExist { + result[key] = value + } + } + var resultToString string + if len(result) == 0 { + result["None"] = "None" + } + for key, value := range result { + resultToString += fmt.Sprintf("%v:%v\n", key, value) + } + response <- resultToString +} + +func (object storage) WRITE() { + object.writeToFileAllData() +} + +func (object storage) readFromFile(key string) string { + object.storageFile.Seek(0, 0) + scaner := bufio.NewScanner(object.storageFile) + for scaner.Scan() { + line := scaner.Text() + splitLine := strings.Split(line, ";") + //splitLine[0] is key; splitLine[1] is val + if splitLine[0] == key { + return splitLine[1] + } + } + return "false" +} + +func (object storage) readFromFileAllData() map[string]string { + object.storageFile.Seek(0, 0) + scaner := bufio.NewScanner(object.storageFile) + result := make(map[string]string) + for scaner.Scan() { + line := scaner.Text() + splitLine := strings.Split(line, ";") + //splitLine[0] is key; splitLine[1] is val + result[splitLine[0]] = splitLine[1] + } + return result +} + +func (object storage) deleteFromFile(key string) bool { + object.storageFile.Seek(0, 0) + scaner := bufio.NewScanner(object.storageFile) + var stringResult string + var deleteBool = false + for scaner.Scan() { + line := strings.Split(scaner.Text(), ";") + if line[0] == key { + stringResult += "" + deleteBool = true + } else { + stringResult += line[0] + ";" + line[1] + "\n" + } + } + if deleteBool { + object.storageFile.Seek(0, 0) + ioutil.WriteFile(string(object.storageFile.Name()), []byte(stringResult), 0755) + } + return deleteBool + +} + +func (object storage) writeToFileAllData() { + allDataInFile := object.readFromFileAllData() + var resultString string + resultMap := object.storageMap + for key, fileValue := range allDataInFile { + _, ok := resultMap[key] + if !ok { + resultMap[key] = fileValue + } + } + for key, value := range resultMap { + resultString += fmt.Sprintf("%v;%v\n", key, value) + } + ioutil.WriteFile(string(object.storageFile.Name()), []byte(resultString), 0755) +} + +type storage struct { + storageMap map[string]string + storageMode string + storageFile *os.File +} + +func main() { + var port string + flag.StringVar(&port, "port", "9090", "listening port") + flag.StringVar(&port, "p", "9090", "listening port") + var host string + flag.StringVar(&host, "h", "0.0.0.0", "listening IP") + flag.StringVar(&host, "host", "0.0.0.0", "listening IP") + var mode string + flag.StringVar(&mode, "m", "memory", "storage mode disk(default) or memory") + flag.StringVar(&mode, "mode", "memory", "storage mode disk(default) or memory") + if mode != "disk" { + mode = "memory" + } + + flag.Parse() + fmt.Println(port, host) + + pwd, _ := os.Getwd() + fmt.Println(pwd) + logFile, _ := os.OpenFile(pwd+"/log/server.log", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0755) + logWriter := log.New(logFile, "", log.Ldate|log.Ltime) + + listener, err := net.Listen("tcp", host+":"+port) + + if err != nil { + fmt.Println(err) + return + } + defer listener.Close() + fmt.Println("Server is listening...") + logWriter.Println("start server at address: " + host + ":" + port) + + request := make(chan string) + response := make(chan string) + go goStorage(request, response, mode) + + for { + conn, err := listener.Accept() + logWriter.Printf("received connection from %v", conn.RemoteAddr()) + if err != nil { + fmt.Println(err) + conn.Close() + continue + } + go handleConnection(conn, request, response, logWriter) // запускаем горутину для обработки запроса + + } + +} + +func handleConnection(conn net.Conn, request chan string, response chan string, logWriter *log.Logger) { + defer conn.Close() + for { + // read data from request + input := make([]byte, 1024*32) + n, err := conn.Read(input) + if n == 0 || err != nil { + fmt.Println("Read error:", err) + log.Printf("received message read error") + break + } + log.Printf("received message from client(%v) %v", conn.RemoteAddr(), string(input[0:n])) + logWriter.Printf("received message from client(%v) '%v'", conn.RemoteAddr(), string(input[0:n])) + request <- string(input[0:n]) + resp := []byte(<-response) + log.Printf("send response to client(%v) %v", conn.RemoteAddr(), string(resp)) + logWriter.Printf("send response to client(%v) %v", conn.RemoteAddr(), string(resp)) + conn.Write(resp) + } +} + +func goStorage(request chan string, response chan string, mode string) { + pwd, _ := os.Getwd() + file, _ := os.OpenFile(pwd+"/common_files/storage", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0775) + defer file.Close() + newStorage := storage{ + make(map[string]string), + mode, + file, + } + requestsCount := 0 + for entry := range request { + entryList := strings.Split(entry, " ") + keyword := strings.ToUpper(entryList[0]) + switch keyword { + case "GET": + if len(entryList) == 2 { + key := string(entryList[1]) + newStorage.GET(key, response) + } else { + response <- "Wrong operator count" //send data to client + } + case "SET": + if len(entryList) == 3 { + key := string(entryList[1]) + value := string(entryList[2]) + newStorage.SET(key, value, response) + } else { + response <- "Wrong operator count" //send data to client + } + case "DEL": + if len(entryList) == 2 { + key := string(entryList[1]) + newStorage.DEL(key, response) + } else { + response <- "Wrong operator count" //send data to client + } + case "KEYS": + if len(entryList) == 2 { + pattern := string(entryList[1]) + pattern = strings.Replace(pattern, "*", "", -1) + newStorage.KEYS(pattern, response) + } else if len(entryList) == 1 { + pattern := ".*" + newStorage.KEYS(pattern, response) + } else { + response <- "keys are not exist" //send data to client + } + default: + response <- "Wrong expression:(" //send data to client + } + requestsCount++ + if requestsCount >= 3 { + requestsCount = 0 + newStorage.WRITE() + } + } +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..4c23424 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "io/ioutil" + "os" + "reflect" + "testing" +) + +func TestStoreMethods(t *testing.T) { + ioutil.WriteFile("storage", []byte(""), 0755) + file, _ := os.OpenFile("storage", os.O_CREATE|os.O_RDWR, 0775) + file.WriteString("bird;Twit\n") + file.Seek(0, 0) + newStorage := storage{ + map[string]string{ + "cat": "Tom", + "dog": "Bob", + "duck": "Donald", + "mouse": "Djery", + "monkey": "Lisa", + "snake": "Ka", + "lion": "Symba", + }, + "disk", + file, + } + + t.Log("GET method of storage") + ch1 := make(chan string, 1) + go newStorage.GET("cat", ch1) + result := <-ch1 + if result == "cat:Tom" { + t.Log("\t[OK]\tShould get Tom") + } else { + t.Error("\t[ERR]\tShould get Tom") + } + + ch2 := make(chan string, 1) + go newStorage.GET("wolf", ch2) + if <-ch2 == "no pair contains key=wolf" { + t.Log("\t[OK]\tStore no contain key dog") + } else { + t.Error("\t[ERR]\tShould Error wrong value return") + } + + ch2s := make(chan string, 1) + go newStorage.GET("bird", ch2s) + if <-ch2s == "bird:Twit" { + t.Log("\t[OK]\tStore no contain key dog") + } else { + t.Error("\t[ERR]\tShould Error wrong value return") + } + + t.Log("SET method of storage") + ch3 := make(chan string, 1) + go newStorage.SET("panda", "Pow", ch3) + if <-ch3 == "pair panda:Pow created" { + t.Log("\t[OK]\tPair panda:Pow inserted in to storage") + } else { + t.Error("\t[ERR]\tStore contains key panda, do not may overwrite it") + } + + ch4 := make(chan string, 1) + go newStorage.SET("cat", "Poor", ch4) + if <-ch4 == "store contains pair with key cat: Tom" { + t.Log("\t[OK]\tStore contains key cat") + } else { + t.Error("\t[ERR]\tPair cat: Poor wrote to storage") + } + + t.Log("DEL method of storage") + ch5 := make(chan string, 1) + go newStorage.DEL("dog", ch5) + if <-ch5 == "pair deleted" { + t.Log("\t[OK]\tKey dog deleted") + } else { + t.Error("\t[ERR]\tNo find key dog in to storage") + } + + ch6 := make(chan string, 1) + go newStorage.DEL("cow", ch6) + if <-ch6 == "no pair for delete" { + t.Log("\t[OK]\tStore don't contains key cow") + } else { + t.Error("\t[ERR]\tKey cow deleted") + } + + t.Log("KEYS method of storage") + ch7 := make(chan string, 1) + go newStorage.KEYS("on", ch7) + res := <-ch7 + //fmt.Println(res, "res") + if reflect.DeepEqual("monkey:Lisa\nlion:Symba\n", res) { + t.Log("\t[OK]\tMethod keys is work") + } else if reflect.DeepEqual("lion:Symba\nmonkey:Lisa\n", res) { + t.Log("\t[OK]\tMethod keys is work") + } else { + t.Error("\t[ERR]\tMethod keys not work") + } + + ch8 := make(chan string, 1) + go newStorage.KEYS("bear", ch8) + if reflect.DeepEqual("None:None\n", <-ch8) { + t.Log("\t[OK]\tMethod keys is work") + } else { + t.Error("\t[ERR]\tMethod keys not work") + } + + t.Log("Write data to file method") + newStorage.writeToFileAllData() + readingData := newStorage.readFromFileAllData() + if reflect.DeepEqual(readingData, newStorage.storageMap) { + t.Log("\t[OK]\tWrite/read file") + } else { + t.Error("\t[ERR]\tKey cow deleted") + } +}