diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cd31cfb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM ubuntu + +EXPOSE 9090 + +WORKDIR /usr/src/ + +COPY ./serv/ /usr/src/serv + +COPY ./client/ /usr/src/client + +COPY ./docker-entrypoint.sh /usr/local/bin + +ENTRYPOINT docker-entrypoint.sh + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..94db37d --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +################ Preparation #################### +REGISTRY := nick1chak +VERSION := $(shell cat VERSION) +DOCKER_IMAGE_SERV := $(REGISTRY)/redis:$(VERSION) +DOCKER_IMAGE_CLI := $(REGISTRY)/red_client:$(VERSION) + +REPOSITORY_PATH := /usr/src/redis +SERVER_PATH := serv +CLIENT_PATH := client + +DOCKER_BUILDER := golang:1.11 +DOCKER_RUNNER = docker run --rm -v $(CURDIR):$(REPOSITORY_PATH) +DOCKER_RUNNER += -w $(REPOSITORY_PATH)/ +################ End Preparation #################### + + + +################ Binary Target #################### +.PHONY: build +build: build_cli build_serv + docker build -t redis:build . + +.PHONY: build_cli +build_cli: + $(DOCKER_RUNNER)$(CLIENT_PATH) $(DOCKER_BUILDER) go build + +.PHONY: build_serv +build_serv: + $(DOCKER_RUNNER)$(SERVER_PATH) $(DOCKER_BUILDER) go build + +############### Docker Target #################### +.PHONY: run +run: build + docker run -it --name redis_build redis:build + +.PHONY: build_check +build_check: + docker build -t redis:checkSR ./$(SERVER_PATH) + docker build -t redis:checkCL ./$(CLIENT_PATH) +################# Testing #################### +.PHONY: check +check: build_check + docker run --rm --name redis_checkSR redis:checkSR + docker run --rm --name redis_checkCL redis:checkCL + +.PHONY: test +test: + $(DOCKER_RUNNER)$(SERVER_PATH) $(DOCKER_BUILDER) go test -cover -coverprofile=coverage.out + +.PHONY: clean +clean: + docker rm -f redis_build &>/dev/null + docker rmi -f redis:build &>/dev/null + docker rmi -f redis:checkSR &>/dev/null + docker rmi -f redis:checkCL &>/dev/null diff --git a/README.md b/README.md new file mode 100644 index 0000000..f045912 --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +Server-client solution for storing KV data, lightweight analog of the Redis + +MAKEFILE: + + build - Compiles the binaries for server and for client and creates the ready-to-use docker container (use make run) + test - Runs Unit and Integration tests for the server. File coverage.out will apeear in the serv dir + check - Runs subsequently "go vet", "goimports", "golint" on the project. Fails if any errors occur. + run - Starts the server and the client with default configuration (port: 9090, storage: RAM) + +serv: + + -p, --port - The port for listening on (default 9090) + -m, --mode - The possible storage option + memory - use RAM ad a storage (default) + disk - use disk as a storage + +client: + + -p, --port - The port for listening on (default 9090) + -h, --host - The host to connect to the server (default: 127.0.0.1) + +commands: + + SET key value - updates one key at a time with the given value. + GET key - returns tuple of the value and the key state. + DEL key - removes one key at a time and returns deleted value. + + + diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..3b04cfb --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.2 diff --git a/client/Dockerfile b/client/Dockerfile new file mode 100644 index 0000000..90bdf86 --- /dev/null +++ b/client/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.11 + +EXPOSE 9090 + +WORKDIR /usr/src/client + +COPY ./ /usr/src/client + +COPY ./docker-entrypoint.sh /usr/local/bin + +RUN go get -u golang.org/x/lint/golint + +RUN go get -u golang.org/x/tools/cmd/goimports + +ENTRYPOINT docker-entrypoint.sh + + diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..e8770ec --- /dev/null +++ b/client/client.go @@ -0,0 +1,77 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "log" + "net" + "os" +) + +//Reading commands from terminal +func startup() (addr string, err error) { + //default values + addr = "" + host := "127.0.0.1" + port := ":9090" + err = nil + + args := os.Args + for i := 1; i < len(args); i += 2 { + switch args[i] { + // -p, --port : set the port for listening on + case "-p", "--port": + if args[i+1][0] == ':' { + port = args[i+1] + } else { + port = ":" + args[i+1] + } + // -m, --mode : enable mirroring data to the drive + case "-h", "--host": + host = args[i+1] + default: + err = errors.New("ERROR: Unknown command") + return + } + } + addr = host + port + return +} + + +func main() { + addr, cmdErr := startup() + if cmdErr == nil { + conn, netErr := net.Dial("tcp", addr) + if netErr != nil { + log.Fatalln(netErr) + } else { + defer conn.Close() + fmt.Println("Connected to", addr) + for { + // read in input from stdin + reader := bufio.NewReader(os.Stdin) + command, _ := reader.ReadString('\n') + fmt.Fprintf(conn, command) + /* + var message string + for { + str, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + fmt.Print(err) + break + } + message = message + str + "\n" + } + */ + message, _ := bufio.NewReader(conn).ReadString('\r') + fmt.Println(string(message)) + } + } + + } else { + //Some CMD error + fmt.Println(cmdErr) + } +} diff --git a/client/docker-entrypoint.sh b/client/docker-entrypoint.sh new file mode 100755 index 0000000..0ddd558 --- /dev/null +++ b/client/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + + +echo client go vet +go vet +echo client golint +golint +echo client goimports +goimports diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..aa2d630 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +echo starting server +./serv/serv & &>/dev/nul +echo 'waiting for cli to start' +sleep 2 +echo starting client +./client/client diff --git a/serv/Dockerfile b/serv/Dockerfile new file mode 100644 index 0000000..eff8a34 --- /dev/null +++ b/serv/Dockerfile @@ -0,0 +1,17 @@ +FROM golang:1.11 + +EXPOSE 9090 + +WORKDIR /usr/src/serv + +COPY ./ /usr/src/serv + +COPY ./docker-entrypoint.sh /usr/local/bin + +RUN go get -u golang.org/x/lint/golint + +RUN go get -u golang.org/x/tools/cmd/goimports + +ENTRYPOINT docker-entrypoint.sh + + diff --git a/serv/docker-entrypoint.sh b/serv/docker-entrypoint.sh new file mode 100755 index 0000000..f74f137 --- /dev/null +++ b/serv/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + + +echo serv go vet +go vet +echo serv golint +golint +echo serv goimports +goimports diff --git a/serv/saveToDisk.go b/serv/saveToDisk.go new file mode 100644 index 0000000..495fbe0 --- /dev/null +++ b/serv/saveToDisk.go @@ -0,0 +1,160 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" +) + + +//Using disk as storage +func saveData(filePath string, clientCh clientChan) { + //Redis disk storage + file, fileErr := os.Create(filePath) + //file, fileErr := os.OpenFile(filePath, os.O_RDWR, 0600) + if fileErr != nil{ + panic(fileErr) + } + defer file.Close() + format := "%" + strconv.Itoa(keyBuffLen) + "s %" + strconv.Itoa(valueBuffLen) + "s\n" + + fmt.Println("Redis started\nDisk mode") + for cmd := range clientCh.input { + cmdList := strings.Fields(cmd) + //check client's input + if !checkInput(&clientCh, &cmdList){ + continue + } + + switch cmdList[0] { + // SET + case "SET": + value, err := setDisk(file, &cmdList, &format) + go func() { + clientCh.output <- value + clientCh.err <- err + }() + // GET + case "GET": + value, err := getDisk(file, &cmdList) + go func() { + clientCh.output <- value + clientCh.err <- err + }() + // DEL + case "DEL": + value, err := delDisk(file, &cmdList, &format) + go func() { + clientCh.output <- value + clientCh.err <- err + }() + //exit + case "stop": + if cmdList[1] == "redis" { + break + } + default: + go func() { + clientCh.output <- "" + err := errors.New("ERROR: Unknown command") + clientCh.err <- err + }() + } + } +} + + +func setDisk(file *os.File, cmdList *[]string, format *string)(value string, err error){ + if len(*cmdList) != 3 { + err := errors.New("ERROR: Wrong command\nSET ") + return "", err + } + key := (*cmdList)[1] + value = (*cmdList)[2] + var offset int64 + _, _, offset, err = findKey(file, &key) + if err != nil { + return "", err + } + + str := fmt.Sprintf(*format, key, value) + buff := []byte(str) + _, err = file.WriteAt(buff, offset) + + if err != nil { + return "", err + } + return value, nil +} + +func getDisk(file *os.File, cmdList *[]string)(value string, err error){ + if len(*cmdList) != 2 { + err := errors.New("ERROR: Wrong command\nGET ") + return "", err + } + key := (*cmdList)[1] + ok, value, _, err := findKey(file, &key) + if err != nil { + return "", err + } + if ok { + //Returning the value + return value, nil + } + //There are no such key in db + err = errors.New("ERROR: Unknown key \"" + key + "\"") + return "", err +} + +func delDisk(file *os.File, cmdList *[]string, format *string)(value string, err error){ + if len(*cmdList) != 2 { + err := errors.New("ERROR: Wrong command\nGET ") + return "", err + } + key := (*cmdList)[1] + ok, value, offset, err := findKey(file, &key) + if err != nil { + return"", err + } + if ok { + //Replacing the key and the value + str := fmt.Sprintf(*format, "", "") + buff := []byte(str) + _, err := file.WriteAt(buff, offset) + return value, err + } + //There are no such key in db + err = errors.New("ERROR: Unknown key \"" + key + "\"") + return"", err + +} + +func findKey(file *os.File, key *string)(ok bool, value string, offset int64, err error){ + reader := bufio.NewReader(file) + for { + file.Seek(offset,0) + str, err := reader.ReadString('\n') + if err != nil{ + if err == io.EOF || str == "\n"{ + err = nil + return ok, value, offset, err + } + return ok, value, offset, err + } + words := strings.Fields(str) + //Skipping empty strings + if len(words) != 2{ + offset += int64(len(str)) + continue + } + if words[0] == *key { + value = words[1] + return true, value, offset, err + } + offset += int64(len(str)) + } +} diff --git a/serv/serv.go b/serv/serv.go new file mode 100644 index 0000000..bf95a0c --- /dev/null +++ b/serv/serv.go @@ -0,0 +1,253 @@ +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "strings" +) + + +//MAX key and value len +const keyBuffLen = 63 //10 +const valueBuffLen = 63 //12 + +//Reading commands from terminal +func startup() (port string, disk bool, filePath string, err error) { + //default values + port = ":9090" + disk = false + //filePath = "/data/redisDatabase" + filePath = "redisDatabase" + //filePath = "text" + err = nil + + args := os.Args + for i := 1; i < len(args); i += 2 { + switch args[i] { + // -p, --port : set the port for listening on + case "-p", "--port": + if args[i+1][0] == ':' { + port = args[i+1] + } else { + port = ":" + args[i+1] + } + // -m, --mode : enable mirroring data to the drive + case "-m", "--mode": + if args[i+1] == "disk" { + disk = true + } else { + if args[i+1] == "memory" { + disk = false + } + } + default: + err = errors.New("ERROR: Unknown command") + return + } + } + return +} + +//Using RAM as storage +func redis(clientCh clientChan) { + fmt.Println("Redis started\nRAM mode") + //Redis RAM storage + var data = make(map[string]string) +Loop: + for cmd := range clientCh.input { + cmdList := strings.Fields(cmd) + //check client's input + if !checkInput(&clientCh, &cmdList){ + continue + } + + switch cmdList[0] { + // SET + case "SET": + value, err := setRAM(&cmdList, &data) + go func() { + clientCh.output <- value + clientCh.err <- err + }() + // GET + case "GET": + value, err := getRAM(&cmdList, &data) + go func() { + clientCh.output <- value + clientCh.err <- err + }() + // DEL + case "DEL": + value, err := delRAM(&cmdList, &data) + go func() { + clientCh.output <- value + clientCh.err <- err + }() + //exit + case "exit": + go func() { + clientCh.output <- "" + err := errors.New("ERROR") + clientCh.err <- err + }() + break Loop + default: + // sending error + go func() { + clientCh.output <- "" + err := errors.New("ERROR: Wrong command\nUse:\n SET \n GET \n DEL ") + clientCh.err <- err + }() + } + + } + fmt.Println("STOP") +} + +// SET +func setRAM(cmdList *[]string, data *(map[string]string))(value string, err error){ + + if len(*cmdList) != 3 { + err := errors.New("ERROR: Wrong command\nSET ") + return "", err + } + (*data)[(*cmdList)[1]] = (*cmdList)[2] + return (*cmdList)[2], nil +} + +// GET +func getRAM(cmdList *[]string, data *(map[string]string))(value string, err error){ + if len(*cmdList) != 2 { + err := errors.New("ERROR: Wrong command\nGET ") + return "", err + } + value, ok := (*data)[(*cmdList)[1]] + if ok { + return value, nil + } + err = errors.New("ERROR: Unknown key \"" + (*cmdList)[1] + "\"") + return "", err +} + +// DEL +func delRAM(cmdList *[]string, data *(map[string]string))(value string, err error){ + if len(*cmdList) != 2 { + err := errors.New("ERROR: Wrong command\nGET ") + return "", err + } + value, ok := (*data)[(*cmdList)[1]] + if ok { + delete(*data, (*cmdList)[1]) + return value, nil + } + err = errors.New("ERROR: Unknown key \"" + (*cmdList)[1] + "\"") + return "", err +} + +//Check client's input +func checkInput(clientCh *clientChan, cmdList *[]string)(bool){ + if len(*cmdList) < 1 || len(*cmdList) > 3{ + go func() { + clientCh.output <- "" + err := errors.New("ERROR: Wrong command\nUse:\n SET \n GET \n DEL ") + clientCh.err <- err + }() + return false + } + if len(*cmdList) == 2 && len((*cmdList)[1]) > keyBuffLen { + go func() { + clientCh.output <- "" + err := errors.New("ERROR: The key is too long") + clientCh.err <- err + }() + return false + } + if len(*cmdList) == 3 && (len((*cmdList)[1]) > keyBuffLen || len((*cmdList)[2]) > valueBuffLen){ + go func() { + clientCh.output <- "" + err := errors.New("ERROR: Key or value is too long") + clientCh.err <- err + }() + return false + } + return true +} + + +//Connection to the client +func handle(conn net.Conn, clientCh clientChan) { + defer conn.Close() + + scanner := bufio.NewScanner(conn) +Loop: + for scanner.Scan() { + command := scanner.Text() + clientCh.input <- command + + answer := <-clientCh.output + err := <-clientCh.err + switch err{ + case nil: + answer += "\r\n" + case errors.New("exit"): + break Loop + default: + answer = err.Error() + "\r\n" + } + io.WriteString(conn, answer) + } +} + +//Clients <-> Redis channels +type clientChan struct { + input chan string + output chan string + err chan error +} + +func main() { + port, disk, filePath, cmdErr := startup() + if cmdErr == nil { + li, netErr := net.Listen("tcp", port) + if netErr != nil { + log.Fatalln(netErr) + } else { + defer li.Close() + + //Prepare channels for client commands and saving data + clInput := make(chan string) + clOutput := make(chan string) + clErr := make(chan error) + clientCh := clientChan { + input: clInput, + output: clOutput, + err: clErr, + } + + if disk { + //Storage - disk + go saveData(filePath, clientCh) + } else { + //Storage - RAM + go redis(clientCh) + } + + for { + conn, err := li.Accept() + if err != nil { + log.Fatalln(err) + } + go handle(conn, clientCh) + } + } + } else { + //Some CMD error + fmt.Println(cmdErr) + } + fmt.Println("stop") +} diff --git a/serv/serv_test.go b/serv/serv_test.go new file mode 100644 index 0000000..b72527f --- /dev/null +++ b/serv/serv_test.go @@ -0,0 +1,123 @@ +package main + +import ( + //"errors" + "testing" +) + + + +func TestRedisRAM(t *testing.T) { + clInput := make(chan string) + clOutput := make(chan string) + clErr := make(chan error) + clientCh := clientChan{ + input: clInput, + output: clOutput, + err: clErr, + } + + + tests := []struct { + cmd string + out string + }{ + {"", "",}, + {"qwe", "",}, + {"GET", "",}, + {"GET qwe", "",}, + {"GET qwe qwe", "",}, + {"GET qwe qwe qwe", "",}, + {"SET", "",}, + {"SET qwe", "",}, + {"SET qwe qwe qwe", "",}, + {"DEL", "",}, + {"DEL qwe", "",}, + {"DEL qwe qwe", "",}, + {"DEL qwe qwe qwe", "",}, + {"SET key1 val1", "val1",}, + {"SET key2 val2", "val2",}, + {"SET key3 val3", "val3",}, + {"SET key4 val4", "val4",}, + {"GET key4 ", "val4",}, + {"GET key2 ", "val2",}, + {"DEL key3 ", "val3",}, + {"DEL key2 ", "val2",}, + {"DEL key3 ", "",}, + {"GET key3 ", "",}, + {"GET key2 ", "",}, + } + go redis(clientCh) + for i, st := range tests { + + f := func(t *testing.T) { + t.Logf("Run test #%d for %s", i, st.cmd) + + clientCh.input <- st.cmd + out := <-clientCh.output + if out != st.out { + t.Errorf("[ERR]:\n\tOutput: want %s, - got %s", st.out, out) + } + } + // Run sub test + t.Run("RAM", f) + } + +} + +func TestRedisDISK(t *testing.T) { + + clInput := make(chan string) + clOutput := make(chan string) + clErr := make(chan error) + clientCh := clientChan{ + input: clInput, + output: clOutput, + err: clErr, + } + + tests := []struct { + cmd string + out string + }{ + {"", "",}, + {"qwe", "",}, + {"GET", "",}, + {"GET qwe", "",}, + {"GET qwe qwe", "",}, + {"GET qwe qwe qwe", "",}, + {"SET", "",}, + {"SET qwe", "",}, + {"SET qwe qwe qwe", "",}, + {"DEL", "",}, + {"DEL qwe", "",}, + {"DEL qwe qwe", "",}, + {"DEL qwe qwe qwe", "",}, + {"SET key1 val1", "val1",}, + {"SET key2 val2", "val2",}, + {"SET key3 val3", "val3",}, + {"SET key4 val4", "val4",}, + {"GET key4 ", "val4",}, + {"GET key2 ", "val2",}, + {"DEL key3 ", "val3",}, + {"DEL key2 ", "val2",}, + {"DEL key3 ", "",}, + {"GET key3 ", "",}, + {"GET key2 ", "",}, + } + go saveData("redisDatabase", clientCh) + for i, st := range tests { + + f := func(t *testing.T) { + t.Logf("Run test #%d for %s", i, st.cmd) + + clientCh.input <- st.cmd + out := <-clientCh.output + if out != st.out { + t.Errorf("[ERR]:\n\tOutput: want %s, - got %s", st.out, out) + } + } + // Run sub test + t.Run("DISK", f) + } +}