From 23822fd7a9f85ab475d978c9e7b84dbc39a3956a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Mon, 11 Jul 2016 22:36:32 +0200 Subject: [PATCH 01/15] init protocol rpc impl --- avro_flupe.avrp | 56 ++++++++++++++++++++++ protocol.go | 118 +++++++++++++++++++++++++++++++++++++++++++++++ protocol_test.go | 20 ++++++++ 3 files changed, 194 insertions(+) create mode 100644 avro_flupe.avrp create mode 100644 protocol.go create mode 100644 protocol_test.go diff --git a/avro_flupe.avrp b/avro_flupe.avrp new file mode 100644 index 0000000..ca855dc --- /dev/null +++ b/avro_flupe.avrp @@ -0,0 +1,56 @@ +{ + "protocol":"AvroSourceProtocol", + "namespace":"org.apache.flume.source.avro", + "doc":"* Licensed to the Apache Software Foundation (ASF).", + "types":[ + { + "type":"enum", + "name":"Status", + "symbols":[ + "OK", + "FAILED", + "UNKNOWN" + ] + }, + { + "type":"record", + "name":"AvroFlumeEvent", + "fields":[ + { + "name":"headers", + "type":{ + "type":"map", + "values":"string" + } + }, + { + "name":"body", + "type":"bytes" + } + ] + } + ], + "messages":{ + "append":{ + "request":[ + { + "name":"event", + "type":"AvroFlumeEvent" + } + ], + "response":"Status" + }, + "appendBatch":{ + "request":[ + { + "name":"events", + "type":{ + "type":"array", + "items":"AvroFlumeEvent" + } + } + ], + "response":"Status" + } + } +} diff --git a/protocol.go b/protocol.go new file mode 100644 index 0000000..9d026fb --- /dev/null +++ b/protocol.go @@ -0,0 +1,118 @@ +package goavro + +import ( + "crypto/md5" + "encoding/json" + "encoding/hex" + "fmt" +) + +type Protocol struct { + Namespace string `json:"namespace"` + Name string `json:"protocol"` + Fullname string `json:"-"` + Doc string `json:"doc"` + Types []ProtocolType `json:"types"` + Messages map[string]ProtocolMessage `json:"messages"` + MD5 string `json:"-"` +} + +type ProtocolType struct { + TypeX string `json:"type"` + Name string `json:"name,omitempty"` + Symbols []string `json:"symbols,omitempty"` + Fields []Field `json:"fields,omitempty"` + Values string `json:"values,omitempty"` + Items string `json:"items,omitempty"` +} + +type Field struct{ + Name string `json:"name"` +// TypeX ProtocolType `json:"type"` +} + +type ProtocolMessage struct { + Doc string `json:"doc,omitmepty"` +// Request []ProtocolType `json:"request"` + Response string `json:"response"` + Errors []string `json:"errors,omitempty"` + One_way bool `json:"one-way,omitempty"` +} + +const proto = ` +{ + "protocol":"AvroSourceProtocol", + "namespace":"org.apache.flume.source.avro", + "doc":"* Licensed to the Apache Software Foundation (ASF).", + "types":[ + { + "type":"enum", + "name":"Status", + "symbols":[ + "OK", + "FAILED", + "UNKNOWN" + ] + }, + { + "type":"record", + "name":"AvroFlumeEvent", + "fields":[ + { + "name":"headers", + "type":{ + "type":"map", + "values":"string" + } + }, + { + "name":"body", + "type":"bytes" + } + ] + } + ], + "messages":{ + "append":{ + "request":[ + { + "name":"event", + "type":"AvroFlumeEvent" + } + ], + "response":"Status" + }, + "appendBatch":{ + "request":[ + { + "name":"events", + "type":{ + "type":"array", + "items":"AvroFlumeEvent" + } + } + ], + "response":"Status" + } + } +} +` +func NewProtocol() (Protocol, error) { + var result Protocol + err := json.Unmarshal([]byte(proto), &result) + + if err!=nil { + return result, err + } + + if len(result.Name)==0 { + err = fmt.Errorf("Protocol must have a non-empty name.") + } else if len(result.Namespace) == 0 { + err = fmt.Errorf("The namespace property must be a string.") + } + result.Fullname = result.Namespace +"." + result.Name + hasher := md5.New() + hasher.Write([]byte(proto)) + result.MD5 = hex.EncodeToString(hasher.Sum(nil)) + return result, err +} diff --git a/protocol_test.go b/protocol_test.go new file mode 100644 index 0000000..2848e86 --- /dev/null +++ b/protocol_test.go @@ -0,0 +1,20 @@ +package goavro + +import ( + "testing" +) + +func TestProtoParse(t *testing.T) { + proto, err := NewProtocol() + if err!=nil { + t.Fatal("%v",err) + } + if "AvroSourceProtocol" !=proto.Name { + t.Errorf("Proto Name not pared; Expected AvroSourceProtocol / actual %#v (%#v", proto.Name, proto) + } + + if len(proto.MD5)==0 { + t.Errorf("Proto MD5 not calculated; actual %#v ", proto) + } + t.Logf("proto %#v", proto) +} From f94a76c808861002cabd6af58ead834524bf9c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Tue, 12 Jul 2016 16:55:31 +0200 Subject: [PATCH 02/15] rewrite ipc --- encoder_test.go | 41 ++++++++ ipc.go | 264 ++++++++++++++++++++++++++++++++++++++++++++++++ ipc_test.go | 47 +++++++++ protocol.go | 5 +- requestor.go | 109 ++++++++++++++++++++ schema.go | 34 +++++++ 6 files changed, 497 insertions(+), 3 deletions(-) create mode 100644 encoder_test.go create mode 100644 ipc.go create mode 100644 ipc_test.go create mode 100644 requestor.go create mode 100644 schema.go diff --git a/encoder_test.go b/encoder_test.go new file mode 100644 index 0000000..32d37f0 --- /dev/null +++ b/encoder_test.go @@ -0,0 +1,41 @@ +package goavro + +import ( + "testing" + "bytes" +) + +const testSchema= ` +{ + "type": "record", + "name": "test", + "fields" : [ + {"name": "a", "type": "long"}, + {"name": "b", "type": "string"} + ] +} +` +func TestEncode(t *testing.T) { + record,err := NewRecord(RecordSchema(testSchema)) + if err != nil { + t.Fatal(err) + } + record.Set("a",int64(27)) + record.Set("b","foo") + + codec, err := NewCodec(testSchema) + if err != nil { + t.Fatal(err) + } + + bb := new(bytes.Buffer) + if err = codec.Encode(bb, record); err !=nil { + t.Fatal(err) + } + actual := bb.Bytes() + expected := []byte("\x36\x06\x66\x6f\x6f") + + if bytes.Compare(actual, expected) != 0 { + t.Errorf("Actual: %#v; Expected: %#v", actual, expected) + } +} diff --git a/ipc.go b/ipc.go new file mode 100644 index 0000000..18f2e7e --- /dev/null +++ b/ipc.go @@ -0,0 +1,264 @@ +// Package avro provides th log driver for forwarding server logs to +// flume endpoints. +package goavro + +import ( + "fmt" + "net/rpc" + "bytes" + "io" +) +type Client struct { + hostname string + extra map[string]interface{} + conn *rpc.Client + codecFlume Codec + codecDocker Codec + codecMeta Codec + codecString Codec + protocol Protocol +} + +const ( + defaultHost = "10.98.80.113" + defaultPort = "63001" + + hostKey = "avro-host" + portKey = "avro-port" +) + +const AVRO_SCHEMA_LITERAL_HEADER = "flume.avro.schema.literal" + + +const stringSchema = ` +{"type": "string"} +` +const flumeSchema = ` +{ + "type": "record", + "name": "AvroFlumeEvent", + "fields": [{ + "name": "headers", + "type": { + "type": "map", + "values": "string" + } + }, { + "name": "body", + "type": "bytes" + }] +} +` + +const recordSchema = ` +{ + "type": "record", + "name": "docker_logs", + "doc:": "A basic schema for storing docker container logs", + "namespace": "docker", + "fields": [ + { + "doc": "Docker container ID", + "type": "string", + "name": "container_id" + }, + { + "doc": "Docker container Name", + "type": "string", + "name": "container_name" + }, + { + "doc": "Docker image ID", + "type": "string", + "name": "image_id" + }, + { + "doc": "Docker image Name", + "type": "string", + "name": "image_name" + }, + { + "doc": "Docker container commmand", + "type": "string", + "name": "command" + }, + { + "doc": "Docker container created timestamp", + "type": "long", + "name": "created" + }, + { + "doc": "Source of log (stdout, stderr)", + "type": "string", + "name": "source" + }, + { + "doc": "Docker host", + "type": "string", + "name": "docker_host" + }, + { + "doc": "Log message", + "type": "string", + "name": "log" + }, + { + "doc": "Unix timestamp in milliseconds", + "type": "long", + "name": "timestamp" + } + ] +} +` + +func NewDefaultClient() (*Client, error) { + return NewClient(defaultHost, defaultPort) + +} +// New create a avro logger using the configuration passed in on +// the context. +func NewClient(host string, port string) (*Client, error) { + servAddr := host+":"+port + conn, err := rpc.Dial("tcp", servAddr) + if err != nil { + return nil, err + } + + + codec, err := NewCodec(flumeSchema) + if err != nil { + return nil, err + } + + codecDocker, err := NewCodec(recordSchema) + if err != nil { + return nil, err + } + + codecMeta, err := NewCodec(metadataSchema) + if err != nil { + return nil, err + } + + codecString, err := NewCodec(stringSchema) + if err != nil { + return nil, err + } + + proto, err := NewProtocol() + if err != nil { + return nil, err + } + + return &Client { + extra: make(map[string]interface{}), + hostname: "TODO", + conn: conn, + codecFlume: codec, + codecDocker: codecDocker, + codecMeta: codecMeta, + codecString: codecString, + protocol: proto, + }, nil +} + +func (a *Client) Write_handshake_request( buffer io.Writer ) (err error) { + local_hash :=a.protocol.MD5 + // remote_name :="" // only setted by handshake response + remote_hash := make([]byte,0) + if len(remote_hash)==0 { + remote_hash = local_hash + } + + record, err := NewRecord(RecordSchema(handshakeRequestshema)) + if err != nil { + return fmt.Errorf("Avro fail to init record handshakeRequest",err) + } + + record.Set("clientHash", local_hash) + record.Set("serverHash", remote_hash) +// record.Set("clientProtocol", a.protocol.Name) + codecHandshake, err := NewCodec(handshakeRequestshema) + if err != nil { + return err + } + + if err = codecHandshake.Encode(buffer, record); err !=nil { + return fmt.Errorf("Encode handshakeRequest ",err) + } + return nil +} + + +func (a *Client) Log(msg string) error { + + bb := new(bytes.Buffer) + a.Write_handshake_request(bb) + + flumeRecord, errFlume := NewRecord(RecordSchema(flumeSchema)) + if errFlume != nil { + return fmt.Errorf("Avro fail to init record",errFlume) + } + headers := make(map[string]interface{}) + headers[AVRO_SCHEMA_LITERAL_HEADER] = stringSchema + headers["host_header"] = "127.0.0.1" + flumeRecord.Set("headers", headers) + flumeRecord.Set("body", []byte("test")) + + + // encode metadata + if err := a.codecMeta.Encode(bb, make(map[string]interface{})); err !=nil { + return fmt.Errorf("Encode metadata ",err) + } + + // encode message name + if err := a.codecString.Encode(bb, "append"); err !=nil { + return fmt.Errorf("Encode message name ",err) + } + + + // encode message parameters + + if err := a.codecFlume.Encode(bb, flumeRecord); err !=nil { + return fmt.Errorf("Encode flumeRecord ",err) + } + + return a.Send(bb.Bytes()) +} + +func (a *Client) Send(bytes []byte ) error { + var reply []byte + err := a.conn.Call("", bytes,&reply) + if err !=nil { + return err + } + return nil +} + +func (a *Client) Close() error { + return a.conn.Close() +} + + +// ValidateLogOpt looks for avro specific log option avro-host avro-port. +func ValidateLogOpt(cfg map[string]string) error { + for key := range cfg { + switch key { + case "env": + case "labes": + case hostKey: + case portKey: + // Accepted + default: + return fmt.Errorf("unknown log opt '%s' for avro log driver", key) + } + } + if len(cfg[hostKey]) == 0 { + cfg[hostKey] = defaultHost + } + if len(cfg[portKey]) == 0 { + cfg[portKey] = defaultPort + } + return nil +} + diff --git a/ipc_test.go b/ipc_test.go new file mode 100644 index 0000000..16dc921 --- /dev/null +++ b/ipc_test.go @@ -0,0 +1,47 @@ +package goavro + +import ( + "testing" + "bytes" +) + + +func TestNewClient(t *testing.T) { + client, err := NewDefaultClient() + if err!=nil { + t.Fatal("%v",err) + } + t.Logf("proto %#v",client) + client.Close() +} + +func TestHandshake(t *testing.T) { + client, err := NewDefaultClient() + if err!=nil { + t.Fatal("%v",err) + } + bb := new(bytes.Buffer) + err = client.Write_handshake_request(bb) + if err !=nil { + t.Fatal("%v", err) + } + t.Logf("handshake message %#v", bb.Bytes()) + client.Send(bb.Bytes()) + client.Close() + +} + + +func TestLog(t *testing.T) { + client, err := NewDefaultClient() + if err!=nil { + t.Fatal("%v",err) + } + err = client.Log("Message test") + if err !=nil { + t.Fatal("%v", err) + } +client.Close() + +} + diff --git a/protocol.go b/protocol.go index 9d026fb..2ae453a 100644 --- a/protocol.go +++ b/protocol.go @@ -3,7 +3,6 @@ package goavro import ( "crypto/md5" "encoding/json" - "encoding/hex" "fmt" ) @@ -14,7 +13,7 @@ type Protocol struct { Doc string `json:"doc"` Types []ProtocolType `json:"types"` Messages map[string]ProtocolMessage `json:"messages"` - MD5 string `json:"-"` + MD5 []byte `json:"-"` } type ProtocolType struct { @@ -113,6 +112,6 @@ func NewProtocol() (Protocol, error) { result.Fullname = result.Namespace +"." + result.Name hasher := md5.New() hasher.Write([]byte(proto)) - result.MD5 = hex.EncodeToString(hasher.Sum(nil)) + result.MD5 = hasher.Sum(nil) return result, err } diff --git a/requestor.go b/requestor.go new file mode 100644 index 0000000..1bb6baa --- /dev/null +++ b/requestor.go @@ -0,0 +1,109 @@ +package goavro + +import ( + "bytes" + "fmt" + "io" + "net" +) + +var REMOTE_HASHES map[string][]byte +var REMOTE_PROTOCOLS map[string]Protocol + + +type Requestor struct { + // Base class for the client side of protocol interaction. + local_protocol Protocol + transport Transport + remote_protocol Protocol + remote_hash []byte + send_protocol bool +} + +func NewRequestor(localProto Protocol, transport Transport) *Requestor { + return &Requestor{ + local_protocol: localProto, + transport: transport, +// remote_protocol: nil, +// remote_hash: nil, +// send_protocol: nil, + } +} + + +func (a *Requestor) RemoteProtocol(proto Protocol) { + a.remote_protocol = proto + REMOTE_PROTOCOLS[a.transport.RemoteName] = proto +} + +func (a *Requestor) RemoteHash(hash []byte) { + a.remote_hash = hash + REMOTE_HASHES[a.transport.RemoteName] = hash +} + +func (a *Requestor) Request(message_name string, request_datum []byte) { + // wrtie a request message and reads a response or error message. + // build handshale and call request + buffer_writer := new(bytes.Buffer) + a.write_handshake_request(buffer_writer) + a.write_call_request(message_name, request_datum, buffer_writer) + + // sen the handshake and call request; block until call response + call_request := buffer_writer.Bytes() + call_response := a.transport.Transceive(call_request) + + // process the handshake and call response + buffer_decoder := new(bytes.Buffer) + if read_handshake_response(buffer_decoder) { + read_call_response(message_name, buffer_decoder) + } else { + a.Request(message_name, request_datum) + } +} + +func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { + local_hash :=a.transport.protocol.MD5 + remote_name := a.transport.RemoteName + remote_hash := make([]byte,0) + if len(remote_hash)==0 { + remote_hash = local_hash + } + + record, err := NewRecord(RecordSchema(handshakeRequestshema)) + if err != nil { + return fmt.Errorf("Avro fail to init record handshakeRequest",err) + } + + record.Set("clientHash", local_hash) + record.Set("serverHash", remote_hash) +// record.Set("clientProtocol", a.protocol.Name) + codecHandshake, err := NewCodec(handshakeRequestshema) + if err != nil { + return err + } + + if err = codecHandshake.Encode(buffer, record); err !=nil { + return fmt.Errorf("Encode handshakeRequest ",err) + } + return nil +} +func (a *Requestor) write_call_request(message_name string, request_datum []byte, buffer io.Writer) { + +} + + + +type Transport struct { + sock *net.Conn + RemoteName string + protocol Protocol +} + +func NewTransport(sock *net.Conn) *Transport{ + return & Transport { + sock: sock, + } +} +func (t *Transport) Transceive(request []byte) []byte{ + +} diff --git a/schema.go b/schema.go new file mode 100644 index 0000000..c221cd4 --- /dev/null +++ b/schema.go @@ -0,0 +1,34 @@ +package goavro + + +const handshakeRequestshema = ` +{ + "type": "record", + "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc", + "fields": [ + {"name": "clientHash", + "type": {"type": "fixed", "name": "MD5", "size": 16}}, + {"name": "clientProtocol", "type": ["null", "string"]}, + {"name": "serverHash", "type": "MD5"}, + {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]} + ] +} +` + +const handshakeResponseshema = ` +{ + "type": "record", + "name": "HandshakeResponse", "namespace": "org.apache.avro.ipc", + "fields": [ + {"name": "match", + "type": {"type": "enum", "name": "HandshakeMatch", + "symbols": ["BOTH", "CLIENT", "NONE"]}}, + {"name": "serverProtocol", + "type": ["null", "string"]}, + {"name": "serverHash", + "type": ["null", {"type": "fixed", "name": "MD5", "size": 16}]}, + {"name": "meta", + "type": ["null", {"type": "map", "values": "bytes"}]} + ] +} +` From cfc068cb34ec8bce67e061edbecf255b2149d2ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Tue, 12 Jul 2016 23:32:13 +0200 Subject: [PATCH 03/15] add new test and continue impl --- protocol.go | 10 +++++++++ protocol_test.go | 15 ++++++++++++++ requestor.go | 54 ++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 70 insertions(+), 9 deletions(-) diff --git a/protocol.go b/protocol.go index 2ae453a..a9b59d2 100644 --- a/protocol.go +++ b/protocol.go @@ -115,3 +115,13 @@ func NewProtocol() (Protocol, error) { result.MD5 = hasher.Sum(nil) return result, err } + +func (p *Protocol) Json() (string, error) { + var result string + bb, err := json.Marshal(p) + if err != nil { + return result, err + + } + return string(bb), nil +} diff --git a/protocol_test.go b/protocol_test.go index 2848e86..e1859f2 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -18,3 +18,18 @@ func TestProtoParse(t *testing.T) { } t.Logf("proto %#v", proto) } + +func TestToJson(t *testing.T) { + protocol, err := NewProtocol() + if err!= nil { + t.Fatal("%#v", err) + } + + json, err := protocol.Json() + if err !=nil { + t.Fatal("%#v", err) + } + if json!= proto { + t.Errorf("Proto to Json not equals; Expected %#v, actual %#v", proto, json) + } +} diff --git a/requestor.go b/requestor.go index 1bb6baa..30134ce 100644 --- a/requestor.go +++ b/requestor.go @@ -10,6 +10,8 @@ import ( var REMOTE_HASHES map[string][]byte var REMOTE_PROTOCOLS map[string]Protocol +var META_WRITER Codec + type Requestor struct { // Base class for the client side of protocol interaction. @@ -53,9 +55,9 @@ func (a *Requestor) Request(message_name string, request_datum []byte) { call_response := a.transport.Transceive(call_request) // process the handshake and call response - buffer_decoder := new(bytes.Buffer) - if read_handshake_response(buffer_decoder) { - read_call_response(message_name, buffer_decoder) + buffer_decoder := bytes.NewBuffer(call_response) + if a.read_handshake_response(buffer_decoder) { + a.read_call_response(message_name, buffer_decoder) } else { a.Request(message_name, request_datum) } @@ -63,10 +65,11 @@ func (a *Requestor) Request(message_name string, request_datum []byte) { func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { local_hash :=a.transport.protocol.MD5 - remote_name := a.transport.RemoteName - remote_hash := make([]byte,0) + remote_name := a.transport.RemoteName + remote_hash := REMOTE_HASHES[remote_name] if len(remote_hash)==0 { remote_hash = local_hash + a.remote_protocol = a.local_protocol } record, err := NewRecord(RecordSchema(handshakeRequestshema)) @@ -76,21 +79,54 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { record.Set("clientHash", local_hash) record.Set("serverHash", remote_hash) -// record.Set("clientProtocol", a.protocol.Name) codecHandshake, err := NewCodec(handshakeRequestshema) if err != nil { return err } + if a.send_protocol { + json, err := a.local_protocol.Json() + if err!=nil { + return err + } + record.Set("clientProtocol", json) + } + if err = codecHandshake.Encode(buffer, record); err !=nil { return fmt.Errorf("Encode handshakeRequest ",err) } return nil } -func (a *Requestor) write_call_request(message_name string, request_datum []byte, buffer io.Writer) { - +func (a *Requestor) write_call_request(message_name string, request_datum []byte, buffer io.Writer) error { + // The format of a call request is: + // * request metadata, a map with values of type bytes + // * the message name, an Avro string, followed by + // * the message parameters. Parameters are serialized according to + // the message's request declaration. + + // TODO request metadata (not yet implemented) + request_metadata := make(map[string]interface{}) + + // encode metadata + if err := META_WRITER.Encode(buffer, make(map[string]interface{})); err !=nil { + return fmt.Errorf("Encode metadata ",err) + } + + message := a.local_protocol.Messages[message_name] + if message==nil { + fmt.Errorf("Unknown message: #{message_name}") + } + buffer.WriteString(message_name) + return nil } +func (a *Requestor) read_handshake_response(decder io.Writer) bool { + return false // TODO +} + +func (a *Requestor) read_call_response(message_name string, decoder io.Writer) { +} + type Transport struct { @@ -105,5 +141,5 @@ func NewTransport(sock *net.Conn) *Transport{ } } func (t *Transport) Transceive(request []byte) []byte{ - + return new(bytes.Buffer).Bytes() } From 2406658c0d3dd209d3255ea6ca13f21e1adae4d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Mon, 1 Aug 2016 14:26:21 +0200 Subject: [PATCH 04/15] reprise travaux --- ipc_test.go | 50 ++++++---------------- protocol.go | 56 ++++++++++++++++++++---- protocol_test.go | 20 +++++++-- requestor.go | 109 +++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 178 insertions(+), 57 deletions(-) diff --git a/ipc_test.go b/ipc_test.go index 16dc921..54e82c2 100644 --- a/ipc_test.go +++ b/ipc_test.go @@ -2,46 +2,24 @@ package goavro import ( "testing" - "bytes" -) - + "net" -func TestNewClient(t *testing.T) { - client, err := NewDefaultClient() - if err!=nil { - t.Fatal("%v",err) - } - t.Logf("proto %#v",client) - client.Close() -} +) -func TestHandshake(t *testing.T) { - client, err := NewDefaultClient() - if err!=nil { - t.Fatal("%v",err) - } - bb := new(bytes.Buffer) - err = client.Write_handshake_request(bb) - if err !=nil { - t.Fatal("%v", err) +func TestRequestor(t *testing.T) { + conn, err := net.Dial("tcp", "10.98.80.113:63001") + if err != nil { + t.Fatal(err) + } + transport := NewTransport(&conn) + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) } - t.Logf("handshake message %#v", bb.Bytes()) - client.Send(bb.Bytes()) - client.Close() - -} + NewRequestor(protocol , *transport) -func TestLog(t *testing.T) { - client, err := NewDefaultClient() - if err!=nil { - t.Fatal("%v",err) - } - err = client.Log("Message test") - if err !=nil { - t.Fatal("%v", err) - } -client.Close() - + response := transport.Transceive([]byte("test")) + t.Log(response) } diff --git a/protocol.go b/protocol.go index a9b59d2..5515111 100644 --- a/protocol.go +++ b/protocol.go @@ -6,12 +6,14 @@ import ( "fmt" ) +var TYPES_CACHE map[string]ProtocolType + type Protocol struct { + Name string `json:"protocol"` Namespace string `json:"namespace"` - Name string `json:"protocol"` Fullname string `json:"-"` Doc string `json:"doc"` - Types []ProtocolType `json:"types"` + Types []AbsType `json:"types"` Messages map[string]ProtocolMessage `json:"messages"` MD5 []byte `json:"-"` } @@ -27,15 +29,21 @@ type ProtocolType struct { type Field struct{ Name string `json:"name"` -// TypeX ProtocolType `json:"type"` + TypeX AbsType `json:"type"` +} + +type AbsType struct { + *ProtocolType + ref string } type ProtocolMessage struct { - Doc string `json:"doc,omitmepty"` -// Request []ProtocolType `json:"request"` - Response string `json:"response"` - Errors []string `json:"errors,omitempty"` - One_way bool `json:"one-way,omitempty"` + Name string `json:"-"` + Doc string `json:"doc,omitempty"` + Request []Field `json:"request"` + Response string `json:"response"` + Errors []string `json:"errors,omitempty"` + One_way bool `json:"one-way,omitempty"` } const proto = ` @@ -96,6 +104,38 @@ const proto = ` } } ` + +func init() { + TYPES_CACHE = make(map[string]ProtocolType) + TYPES_CACHE["bytes"] = ProtocolType{Name:"bytes", TypeX:"bytes"} +} +func (t *AbsType) UnmarshalJSON(data []byte) error { + var nameType string + var protocolType ProtocolType + if err := json.Unmarshal(data, &nameType); err==nil { + protoType, ok := TYPES_CACHE[nameType] + if ok { + t.ref = nameType + t.ProtocolType = &protoType + } else { + return fmt.Errorf("Type %s not found on protocol type cache %#v", data, TYPES_CACHE) + } + } else if err := json.Unmarshal(data, &protocolType); err!=nil { + return fmt.Errorf("Fail to Parse AbsType, %s %s", data,err ) + } + t.ProtocolType = &protocolType + TYPES_CACHE[protocolType.Name] = protocolType + return nil +} + +func (t *AbsType) MarshalJSON()([]byte, error) { + if len(t.ref)>0 { + return json.Marshal(t.ref) + } else { + return json.Marshal(t.ProtocolType) + } +} + func NewProtocol() (Protocol, error) { var result Protocol err := json.Unmarshal([]byte(proto), &result) diff --git a/protocol_test.go b/protocol_test.go index e1859f2..f295997 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -2,12 +2,14 @@ package goavro import ( "testing" + "bytes" + "encoding/json" ) func TestProtoParse(t *testing.T) { proto, err := NewProtocol() if err!=nil { - t.Fatal("%v",err) + t.Fatal(err) } if "AvroSourceProtocol" !=proto.Name { t.Errorf("Proto Name not pared; Expected AvroSourceProtocol / actual %#v (%#v", proto.Name, proto) @@ -16,20 +18,30 @@ func TestProtoParse(t *testing.T) { if len(proto.MD5)==0 { t.Errorf("Proto MD5 not calculated; actual %#v ", proto) } + if len(proto.Types)!=2 { t.Errorf("Types not parsed; Expect 2, actual %i", len(proto.Types)) } t.Logf("proto %#v", proto) } + +func jsonCompact(in string) (out string) { + var json_bytes = []byte(in) + buffer := new(bytes.Buffer) + json.Compact(buffer, json_bytes) + out = buffer.String() + return +} + func TestToJson(t *testing.T) { protocol, err := NewProtocol() if err!= nil { t.Fatal("%#v", err) } - json, err := protocol.Json() + result, err := protocol.Json() if err !=nil { t.Fatal("%#v", err) } - if json!= proto { - t.Errorf("Proto to Json not equals; Expected %#v, actual %#v", proto, json) + if result!= jsonCompact(proto) { + t.Errorf("Proto to Json not equals; Expected %#v, actual %#v",jsonCompact(proto), result) } } diff --git a/requestor.go b/requestor.go index 30134ce..bf3b9e5 100644 --- a/requestor.go +++ b/requestor.go @@ -5,13 +5,15 @@ import ( "fmt" "io" "net" + "log" + "os" ) var REMOTE_HASHES map[string][]byte var REMOTE_PROTOCOLS map[string]Protocol var META_WRITER Codec - +var HANDSHAKE_REQUESTOR_READER Codec type Requestor struct { // Base class for the client side of protocol interaction. @@ -21,6 +23,18 @@ type Requestor struct { remote_hash []byte send_protocol bool } +func init() { + var err error + HANDSHAKE_REQUESTOR_READER, err = NewCodec(handshakeResponseshema) + if err!=nil { + log.Fatal(err) + } + META_WRITER, err = NewCodec(metadataSchema) + if err!=nil { + log.Fatal(err) + } + +} func NewRequestor(localProto Protocol, transport Transport) *Requestor { return &Requestor{ @@ -43,7 +57,7 @@ func (a *Requestor) RemoteHash(hash []byte) { REMOTE_HASHES[a.transport.RemoteName] = hash } -func (a *Requestor) Request(message_name string, request_datum []byte) { +func (a *Requestor) Request(message_name string, request_datum []byte) error { // wrtie a request message and reads a response or error message. // build handshale and call request buffer_writer := new(bytes.Buffer) @@ -56,11 +70,15 @@ func (a *Requestor) Request(message_name string, request_datum []byte) { // process the handshake and call response buffer_decoder := bytes.NewBuffer(call_response) - if a.read_handshake_response(buffer_decoder) { + ok, err := a.read_handshake_response(buffer_decoder) + if err!=nil { + return err + } else if ok { a.read_call_response(message_name, buffer_decoder) } else { a.Request(message_name, request_datum) } + return nil } func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { @@ -97,6 +115,7 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { } return nil } + func (a *Requestor) write_call_request(message_name string, request_datum []byte, buffer io.Writer) error { // The format of a call request is: // * request metadata, a map with values of type bytes @@ -108,23 +127,94 @@ func (a *Requestor) write_call_request(message_name string, request_datum []byte request_metadata := make(map[string]interface{}) // encode metadata - if err := META_WRITER.Encode(buffer, make(map[string]interface{})); err !=nil { + if err := META_WRITER.Encode(buffer, request_metadata); err !=nil { return fmt.Errorf("Encode metadata ",err) } - message := a.local_protocol.Messages[message_name] - if message==nil { + message, found := a.local_protocol.Messages[message_name] + if !found { fmt.Errorf("Unknown message: #{message_name}") } - buffer.WriteString(message_name) + fmt.Fprint(buffer, message_name) + return nil + return a.write_request(message.Request[0].TypeX, request_datum , buffer) } -func (a *Requestor) read_handshake_response(decder io.Writer) bool { - return false // TODO +func (a *Requestor) write_request(request_schema AbsType, request_datum []byte, buffer io.Writer) (err error) { + + codec, err := NewCodec(flumeSchema) + if err !=nil { + return + } + if err = codec.Encode(buffer, request_datum); err != nil { + return + } + return nil +} + +func (a *Requestor) read_handshake_response(decoder io.Reader) (bool, error) { + datum, err := HANDSHAKE_REQUESTOR_READER.Decode(decoder) + if err != nil { + return false, err + } + + record, ok := datum.(*Record) + if !ok { + return false, fmt.Errorf("Fail to decode handshake %T", datum) + } + + var we_have_matching_schema =false + match, err := record.Get("match") + if err!= nil { + return false, err + } + switch match { + case "BOTH": + a.send_protocol = false + we_have_matching_schema =true + case "CLIENT": + err = fmt.Errorf("Handshake failure. match == CLIENT") + if a.send_protocol { + field , err := record.Get("serverProtocol") + if err!= nil { + return false, err + } + a.remote_protocol = field.(Protocol) + field, err = record.Get("serverHash") + if err!= nil { + return false, err + } + a.remote_hash = field.([]byte) + + a.send_protocol = false + we_have_matching_schema = true + } + case "NONE": + err = fmt.Errorf("Handshake failure. match == NONE") + if a.send_protocol { + field , err := record.Get("serverProtocol") + if err!= nil { + return false, err + } + a.remote_protocol = field.(Protocol) + field, err = record.Get("serverHash") + if err!= nil { + return false, err + } + a.remote_hash = field.([]byte) + + a.send_protocol = true + } + default: + err = fmt.Errorf("Unexpected match: #{match}") + } + + return we_have_matching_schema, nil } func (a *Requestor) read_call_response(message_name string, decoder io.Writer) { + } @@ -141,5 +231,6 @@ func NewTransport(sock *net.Conn) *Transport{ } } func (t *Transport) Transceive(request []byte) []byte{ + fmt.Fprintf(os.Stdout, "Transceive %s", request) return new(bytes.Buffer).Bytes() } From 02da71ee49d1e856ae989f40a516ebacece16596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Wed, 24 Aug 2016 09:20:59 +0200 Subject: [PATCH 05/15] good request with hardcoded protocol MD5 --- .gitignore | 1 + Transceiver.go | 70 +++++++++++++++++++ Transceiver_test.go | 28 ++++++++ ipc_test.go | 30 ++++++--- protocol.go | 95 +++++++++----------------- protocol_test.go | 65 +++++++++++++++++- requestor.go | 161 +++++++++++++++++++++++++------------------- requestor_test.go | 127 ++++++++++++++++++++++++++++++++++ schema.go | 5 +- 9 files changed, 439 insertions(+), 143 deletions(-) create mode 100644 Transceiver.go create mode 100644 Transceiver_test.go create mode 100644 requestor_test.go diff --git a/.gitignore b/.gitignore index 9ed3b07..ada62b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.test +.idea diff --git a/Transceiver.go b/Transceiver.go new file mode 100644 index 0000000..fa9296f --- /dev/null +++ b/Transceiver.go @@ -0,0 +1,70 @@ +package goavro + +import ( + "bytes" + "net" + "encoding/binary" + "fmt" +) + +type Transceiver interface { + Transceive(request []bytes.Buffer) ([]byte, error) + RemoteName() string + SetRemoteName(string) +} + + +type NettyTransceiver struct { + sock *net.TCPConn + remoteName string +} +func NewNettyTransceiver(sock *net.TCPConn) Transceiver{ + return NettyTransceiver { + sock: sock, + } +} +func (t NettyTransceiver) RemoteName() string { + return t.remoteName + return t.remoteName + return t.remoteName +} + +func (t NettyTransceiver) SetRemoteName(name string) { + t.remoteName = name +} + +func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]byte, error){ + nettyFrame := new(bytes.Buffer) + t.Pack(nettyFrame, requests) + + // Send request + _, err := t.sock.Write(nettyFrame.Bytes()) + if err!=nil { + return nil, fmt.Errorf("Fail to write on socket") + } + //sfmt.Fprintf(os.Stdout, "BufferSize %v", nettyFrame) + // Read Response + bodyBytes := make([]byte, 1024) + t.sock.Read(bodyBytes) + return bodyBytes, nil +} + +func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { + // Set Netty Serial + + nettySerial :=make([]byte, 4) + binary.BigEndian.PutUint32(nettySerial, uint32(1)) + frame.Write(nettySerial) + + + nettySizeBuffer :=make([]byte, 4) + binary.BigEndian.PutUint32(nettySizeBuffer, uint32(len(requests))) + frame.Write(nettySizeBuffer) + + for _, request := range requests { + requestSize :=make([]byte, 4) + binary.BigEndian.PutUint32(requestSize, uint32(request.Len())) + frame.Write(requestSize) + frame.Write(request.Bytes()) + } +} \ No newline at end of file diff --git a/Transceiver_test.go b/Transceiver_test.go new file mode 100644 index 0000000..71e6e2b --- /dev/null +++ b/Transceiver_test.go @@ -0,0 +1,28 @@ +package goavro + +import ( + "testing" + + "bytes" + "reflect" +) + +func TestPack(t *testing.T) { + transceiver := new(NettyTransceiver) + frame := new(bytes.Buffer) + transceiver.Pack(frame,*new([]bytes.Buffer)) + if frame.Len()!=8 { + t.Fatalf("Frame not equals to 8: %x", frame.Len()) + } + reflect.DeepEqual(frame.Bytes(), []byte("\x00\x00\x00\x01\x00\x00\x00\x00")) + frame.Reset() + + requests:= make([]bytes.Buffer, 2) + requests[0] = *bytes.NewBuffer([]byte("buf1")) + requests[1] = *bytes.NewBuffer([]byte("buf2xxxx")) + transceiver.Pack(frame, requests) + expectedSize:= 8+ requests[0].Len()+4 + requests[1].Len() + 4 + if frame.Len()!= expectedSize { + t.Fatalf("Frame not equals to %x: %x / %x",expectedSize, frame.Len(), frame.Bytes()) + } +} diff --git a/ipc_test.go b/ipc_test.go index 54e82c2..6aeb93c 100644 --- a/ipc_test.go +++ b/ipc_test.go @@ -7,19 +7,33 @@ import ( ) func TestRequestor(t *testing.T) { - conn, err := net.Dial("tcp", "10.98.80.113:63001") + //t.SkipNow() + rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") + conn, err := net.DialTCP("tcp", nil, rAddr) if err != nil { t.Fatal(err) } - transport := NewTransport(&conn) + defer conn.Close() + + transceiver := NewNettyTransceiver(conn) protocol, err := NewProtocol() - if err != nil { - t.Fatal(err) - } + if err != nil { + t.Fatal(err) + } - NewRequestor(protocol , *transport) + flumeRecord, errFlume := protocol.getNewRecord("AvroFlumeEvent") + if errFlume != nil { + t.Fatal(errFlume) + } + headers := make(map[string]interface{}) + headers["host_header"] = "127.0.0.1" + flumeRecord.Set("headers", headers) + flumeRecord.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) + requestor := NewRequestor(protocol, transceiver) + err = requestor.Request("append", flumeRecord) - response := transport.Transceive([]byte("test")) - t.Log(response) + if err != nil { + t.Fatal("Request: ", err) + } } diff --git a/protocol.go b/protocol.go index 5515111..60421c1 100644 --- a/protocol.go +++ b/protocol.go @@ -34,80 +34,27 @@ type Field struct{ type AbsType struct { *ProtocolType - ref string + ref string `json:"-"` } type ProtocolMessage struct { Name string `json:"-"` Doc string `json:"doc,omitempty"` - Request []Field `json:"request"` + Request []Field `json:"request"` Response string `json:"response"` Errors []string `json:"errors,omitempty"` One_way bool `json:"one-way,omitempty"` } const proto = ` -{ - "protocol":"AvroSourceProtocol", - "namespace":"org.apache.flume.source.avro", - "doc":"* Licensed to the Apache Software Foundation (ASF).", - "types":[ - { - "type":"enum", - "name":"Status", - "symbols":[ - "OK", - "FAILED", - "UNKNOWN" - ] - }, - { - "type":"record", - "name":"AvroFlumeEvent", - "fields":[ - { - "name":"headers", - "type":{ - "type":"map", - "values":"string" - } - }, - { - "name":"body", - "type":"bytes" - } - ] - } - ], - "messages":{ - "append":{ - "request":[ - { - "name":"event", - "type":"AvroFlumeEvent" - } - ], - "response":"Status" - }, - "appendBatch":{ - "request":[ - { - "name":"events", - "type":{ - "type":"array", - "items":"AvroFlumeEvent" - } - } - ], - "response":"Status" - } - } -} +{"protocol":"AvroSourceProtocol","namespace":"org.apache.flume.source.avro","doc":"* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations\n * under the License.","types":[{"type":"enum","name":"Status","symbols":["OK","FAILED","UNKNOWN"]},{"type":"record","name":"AvroFlumeEvent","fields":[{"name":"headers","type":{"type":"map","values":"string"}},{"name":"body","type":"bytes"}]}],"messages":{"append":{"request":[{"name":"event","type":"AvroFlumeEvent"}],"response":"Status"},"appendBatch":{"request":[{"name":"events","type":{"type":"array","items":"AvroFlumeEvent"}}],"response":"Status"}}} ` func init() { TYPES_CACHE = make(map[string]ProtocolType) - TYPES_CACHE["bytes"] = ProtocolType{Name:"bytes", TypeX:"bytes"} + TYPES_CACHE["bytes"] = ProtocolType{Name:"bytes"} + TYPES_CACHE["enum"] = ProtocolType{Name:"enum"} + TYPES_CACHE["record"] = ProtocolType{Name:"record"} } func (t *AbsType) UnmarshalJSON(data []byte) error { var nameType string @@ -116,15 +63,15 @@ func (t *AbsType) UnmarshalJSON(data []byte) error { protoType, ok := TYPES_CACHE[nameType] if ok { t.ref = nameType - t.ProtocolType = &protoType + protocolType = protoType } else { return fmt.Errorf("Type %s not found on protocol type cache %#v", data, TYPES_CACHE) } } else if err := json.Unmarshal(data, &protocolType); err!=nil { return fmt.Errorf("Fail to Parse AbsType, %s %s", data,err ) } - t.ProtocolType = &protocolType - TYPES_CACHE[protocolType.Name] = protocolType + t.ProtocolType = &protocolType + TYPES_CACHE[protocolType.Name] = protocolType return nil } @@ -137,7 +84,7 @@ func (t *AbsType) MarshalJSON()([]byte, error) { } func NewProtocol() (Protocol, error) { - var result Protocol + var result Protocol err := json.Unmarshal([]byte(proto), &result) if err!=nil { @@ -165,3 +112,25 @@ func (p *Protocol) Json() (string, error) { } return string(bb), nil } + +func (p *Protocol) getMessageRequestCodec(messageName string) (Codec, error) { + json, err := p.getMessageRequestJson(messageName) + if err!= nil { + return nil, err + } + return NewCodec(json) +} +func (p *Protocol) getMessageRequestJson(messageName string) (string, error) { + field := p.Messages[messageName].Request[0] + avroType := TYPES_CACHE[field.TypeX.ref] + json, err := json.Marshal(avroType) + return string(json), err +} +func (p *Protocol) getNewRecord(typeName string) (*Record, error) { + avroType := TYPES_CACHE[typeName] + json, err := json.Marshal(avroType) + if err!= nil { + return nil, err + } + return NewRecord(RecordSchema(string(json))) +} diff --git a/protocol_test.go b/protocol_test.go index f295997..d74b0a0 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -4,8 +4,19 @@ import ( "testing" "bytes" "encoding/json" + "crypto/md5" + "reflect" ) +func TestMD5(t *testing.T) { + + + + + t.Logf("golang md5 hex %x", md5.Sum([]byte(`{"protocol":"AvroSourceProtocol","namespace":"org.apache.flume.source.avro","doc":"* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations\n * under the License.","types":[{"type":"enum","name":"Status","symbols":["OK","FAILED","UNKNOWN"]},{"type":"record","name":"AvroFlumeEvent","fields":[{"name":"headers","type":{"type":"map","values":"string"}},{"name":"body","type":"bytes"}]}],"messages":{"append":{"request":[{"name":"event","type":"AvroFlumeEvent"}],"response":"Status"},"appendBatch":{"request":[{"name":"events","type":{"type":"array","items":"AvroFlumeEvent"}}],"response":"Status"}}}`))) +} + + func TestProtoParse(t *testing.T) { proto, err := NewProtocol() if err!=nil { @@ -18,8 +29,33 @@ func TestProtoParse(t *testing.T) { if len(proto.MD5)==0 { t.Errorf("Proto MD5 not calculated; actual %#v ", proto) } + if len(proto.Types)!=2 { t.Errorf("Types not parsed; Expect 2, actual %i", len(proto.Types)) } - t.Logf("proto %#v", proto) + + if len(proto.Messages)!=2 { + t.Errorf("Message not parsed: Expected2 , actual %i", len(proto.Messages)) + } + + message, ok := proto.Messages["append"] + if !ok { + t.Errorf("Message append not found : %v", proto.Messages) + } + if len(message.Request) !=1 { + t.Errorf("Message request not equals to 1: %v", message.Request) + } + if message.Request[0].Name!="event" { + t.Errorf("Request parameter not equals event / %v", message.Request[0].Name) + } + if message.Request[0].TypeX.ref!="AvroFlumeEvent" { + t.Errorf("Request parameter type not equals AvroFlumeEvent / %v", message.Request[0].TypeX.ref) + } + typeFound, found := TYPES_CACHE[message.Request[0].TypeX.ref] + if !found { + t.Errorf("Type not found on cache %v", message.Request[0].TypeX.ref) + } + if !reflect.DeepEqual(typeFound, *message.Request[0].TypeX.ProtocolType) { + t.Errorf("Type not equals with cache %v / %v",typeFound, *message.Request[0].TypeX.ProtocolType) + } } @@ -45,3 +81,30 @@ func TestToJson(t *testing.T) { t.Errorf("Proto to Json not equals; Expected %#v, actual %#v",jsonCompact(proto), result) } } + +func TestGetCodec(t *testing.T) { + protocol, err := NewProtocol() + if err!= nil { + t.Fatal("%#v", err) + } + + + flumeRecord, errFlume := protocol.getNewRecord("AvroFlumeEvent") + if errFlume != nil { + t.Fatal(errFlume) + } + headers := make(map[string]interface{}) + headers[AVRO_SCHEMA_LITERAL_HEADER] = stringSchema + headers["host_header"] = "127.0.0.1" + flumeRecord.Set("headers", headers) + flumeRecord.Set("body", []byte("test")) + bb := new(bytes.Buffer) + codec, err := protocol.getMessageRequestCodec("append") + if err != nil { + t.Fatal(err) + } + codec.Encode(bb, flumeRecord) + + t.Logf("AvroFlumeEvent test: %v", bb) + +} diff --git a/requestor.go b/requestor.go index bf3b9e5..9e17632 100644 --- a/requestor.go +++ b/requestor.go @@ -4,21 +4,24 @@ import ( "bytes" "fmt" "io" - "net" + "io/ioutil" "log" - "os" ) var REMOTE_HASHES map[string][]byte var REMOTE_PROTOCOLS map[string]Protocol +var BUFFER_HEADER_LENGTH = 4 +var BUFFER_SIZE = 8192 + var META_WRITER Codec +var META_READER Codec var HANDSHAKE_REQUESTOR_READER Codec type Requestor struct { // Base class for the client side of protocol interaction. local_protocol Protocol - transport Transport + transceiver Transceiver remote_protocol Protocol remote_hash []byte send_protocol bool @@ -33,13 +36,17 @@ func init() { if err!=nil { log.Fatal(err) } + META_READER, err = NewCodec(metadataSchema) + if err!=nil { + log.Fatal(err) + } } -func NewRequestor(localProto Protocol, transport Transport) *Requestor { +func NewRequestor(localProto Protocol, transceiver Transceiver) *Requestor { return &Requestor{ local_protocol: localProto, - transport: transport, + transceiver: transceiver, // remote_protocol: nil, // remote_hash: nil, // send_protocol: nil, @@ -49,42 +56,60 @@ func NewRequestor(localProto Protocol, transport Transport) *Requestor { func (a *Requestor) RemoteProtocol(proto Protocol) { a.remote_protocol = proto - REMOTE_PROTOCOLS[a.transport.RemoteName] = proto + REMOTE_PROTOCOLS[a.transceiver.RemoteName()] = proto } func (a *Requestor) RemoteHash(hash []byte) { a.remote_hash = hash - REMOTE_HASHES[a.transport.RemoteName] = hash + REMOTE_HASHES[a.transceiver.RemoteName()] = hash } -func (a *Requestor) Request(message_name string, request_datum []byte) error { +func (a *Requestor) Request(message_name string, request_datum interface{}) error { // wrtie a request message and reads a response or error message. // build handshale and call request - buffer_writer := new(bytes.Buffer) - a.write_handshake_request(buffer_writer) - a.write_call_request(message_name, request_datum, buffer_writer) + frame1 := new(bytes.Buffer) + frame2 := new(bytes.Buffer) - // sen the handshake and call request; block until call response - call_request := buffer_writer.Bytes() - call_response := a.transport.Transceive(call_request) + err := a.write_handshake_request(frame1) + if err!=nil { + return err + } - // process the handshake and call response - buffer_decoder := bytes.NewBuffer(call_response) - ok, err := a.read_handshake_response(buffer_decoder) + err = a.write_call_requestHeader(message_name, frame1) + if err!=nil { + return err + } + err = a.write_call_request(message_name, request_datum, frame2) if err!=nil { return err - } else if ok { - a.read_call_response(message_name, buffer_decoder) - } else { - a.Request(message_name, request_datum) } + + // sen the handshake and call request; block until call response + buffer_writers := []bytes.Buffer{*frame1, *frame2} + decoder, err := a.transceiver.Transceive(buffer_writers) + if err!=nil { + return err + } + buffer_decoder := bytes.NewBuffer(decoder) + // process the handshake and call response + //ok, err := a.read_handshake_response(buffer_decoder) + fmt.Sprintf("Response %v", buffer_decoder) + //if err!=nil { + // return err + //} else if ok { + // a.read_call_response(message_name, buffer_decoder) + //} else { + // a.Request(message_name, request_datum) + //} return nil } func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { - local_hash :=a.transport.protocol.MD5 - remote_name := a.transport.RemoteName - remote_hash := REMOTE_HASHES[remote_name] + //local_hash :=a.local_protocol.MD5 + + local_hash :=[]byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65") + remote_name := a.remote_protocol.Name + remote_hash := REMOTE_HASHES[remote_name] if len(remote_hash)==0 { remote_hash = local_hash a.remote_protocol = a.local_protocol @@ -92,14 +117,14 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { record, err := NewRecord(RecordSchema(handshakeRequestshema)) if err != nil { - return fmt.Errorf("Avro fail to init record handshakeRequest",err) + return fmt.Errorf("Avro fail to init record handshakeRequest %v",err) } record.Set("clientHash", local_hash) record.Set("serverHash", remote_hash) codecHandshake, err := NewCodec(handshakeRequestshema) if err != nil { - return err + return fmt.Errorf("Avro fail to get codec handshakeRequest %v",err) } if a.send_protocol { @@ -113,50 +138,58 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { if err = codecHandshake.Encode(buffer, record); err !=nil { return fmt.Errorf("Encode handshakeRequest ",err) } + return nil } -func (a *Requestor) write_call_request(message_name string, request_datum []byte, buffer io.Writer) error { - // The format of a call request is: - // * request metadata, a map with values of type bytes - // * the message name, an Avro string, followed by - // * the message parameters. Parameters are serialized according to - // the message's request declaration. +func (a *Requestor) write_call_request(message_name string, request_datum interface{}, frame io.Writer) (err error) { + codec, err := a.local_protocol.getMessageRequestCodec(message_name) + + if err != nil { + return fmt.Errorf("fail to get codec for message %s: %v", message_name, err) + } + a.write_request(codec, request_datum, frame) + return err +} - // TODO request metadata (not yet implemented) +func (a *Requestor) write_call_requestHeader(message_name string, frame1 io.Writer) error { + // The format of a call request is: + // * request metadata, a map with values of type bytes + // * the message name, an Avro string, followed by + // * the message parameters. Parameters are serialized according to + // the message's request declaration. + + // TODO request metadata (not yet implemented) request_metadata := make(map[string]interface{}) + metaBuffer := new(bytes.Buffer) + // encode metadata + if err := META_WRITER.Encode(metaBuffer, request_metadata); err != nil { + return fmt.Errorf("Encode metadata ", err) + } + longCodec.Encode(frame1, int64(metaBuffer.Len())) + frame1.Write(metaBuffer.Bytes()) + longCodec.Encode(frame1, int64(0)) - // encode metadata - if err := META_WRITER.Encode(buffer, request_metadata); err !=nil { - return fmt.Errorf("Encode metadata ",err) - } - - message, found := a.local_protocol.Messages[message_name] - if !found { - fmt.Errorf("Unknown message: #{message_name}") - } - fmt.Fprint(buffer, message_name) + stringCodec.Encode(frame1,message_name) return nil - return a.write_request(message.Request[0].TypeX, request_datum , buffer) } -func (a *Requestor) write_request(request_schema AbsType, request_datum []byte, buffer io.Writer) (err error) { +func (a *Requestor) write_request(request_codec Codec, request_datum interface{}, buffer io.Writer) error { - codec, err := NewCodec(flumeSchema) - if err !=nil { - return - } - if err = codec.Encode(buffer, request_datum); err != nil { - return + + if err := request_codec.Encode(buffer, request_datum); err != nil { + return fmt.Errorf("Fail to encode request_datum %v", err) } return nil } func (a *Requestor) read_handshake_response(decoder io.Reader) (bool, error) { + resp, _ := ioutil.ReadAll(decoder) datum, err := HANDSHAKE_REQUESTOR_READER.Decode(decoder) if err != nil { - return false, err + + return false,fmt.Errorf("Fail to decode %v with error %v", resp, err) } record, ok := datum.(*Record) @@ -214,23 +247,15 @@ func (a *Requestor) read_handshake_response(decoder io.Reader) (bool, error) { } func (a *Requestor) read_call_response(message_name string, decoder io.Writer) { - + // The format of a call response is: + // * response metadata, a map with values of type bytes + // * a one-byte error flag boolean, followed by either: + // * if the error flag is false, + // the message response, serialized per the message's response schema. + // * if the error flag is true, + // the error, serialized per the message's error union schema. +// META_READER.Decode(decoder) } -type Transport struct { - sock *net.Conn - RemoteName string - protocol Protocol -} - -func NewTransport(sock *net.Conn) *Transport{ - return & Transport { - sock: sock, - } -} -func (t *Transport) Transceive(request []byte) []byte{ - fmt.Fprintf(os.Stdout, "Transceive %s", request) - return new(bytes.Buffer).Bytes() -} diff --git a/requestor_test.go b/requestor_test.go new file mode 100644 index 0000000..516e741 --- /dev/null +++ b/requestor_test.go @@ -0,0 +1,127 @@ +package goavro + +import ( + "testing" + "net" + "bytes" + "reflect" +) + +func TestWrite_handshake_request(t *testing.T) { + //t.SkipNow() + rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") + conn, err := net.DialTCP("tcp", nil, rAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + transceiver := NewNettyTransceiver(conn) + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + bb := new(bytes.Buffer) + requestor.write_handshake_request(bb) + // conn.Write(bb.Bytes()) + t.Logf("Handshake_request size %v %x\n",bb.Len(), bb.Bytes()) + t.Logf( "Handshake_request %v\n", bb.String()) + + refHandshake := []byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x00\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x02") + //bytes := bb.Bytes() + //if !reflect.DeepEqual(refHandshake, bytes) { + // t.Fatalf("Handshake not equals to ref %n%x, %n%x", len(refHandshake), refHandshake, len(bytes), bytes) + //} + + + codecHandshake, err := NewCodec(handshakeRequestshema) + if err != nil { + t.Fatal(err) + } + record, err := codecHandshake.Decode(bytes.NewBuffer(refHandshake)) + if err != nil { + t.Fatal(err) + } + t.Logf("\nHandshake_request decoded %v\n", record) + +} + + +func TestWrite_call_request(t *testing.T) { + //t.SkipNow() + rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") + conn, err := net.DialTCP("tcp", nil, rAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + transceiver := NewNettyTransceiver(conn) + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + bb := new(bytes.Buffer) + datum, err := protocol.getNewRecord("AvroFlumeEvent") + if err != nil { + t.Fatal(err) + } + + headers := make(map[string]interface{}) + headers["host_header"] = "127.0.0.1" + datum.Set("headers", headers) + datum.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) + + + + requestor.write_call_request("append", datum,bb) + // conn.Write(bb.Bytes()) + t.Logf("\nCall_request size %v %v\n",bb.Len(), bb.Bytes()) + t.Logf("\nCall_request %v\n", bb.String()) + + codec, err := protocol.getMessageRequestCodec("append") + if err != nil { + t.Fatal(err) + } + value, err := codec.Decode(bb) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(datum, value) { + t.Fatalf("Request not equals to ref %x, %x", datum, value) + } +} + +func TestWrite_call_requestHeader(t *testing.T) { + //t.SkipNow() + rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") + conn, err := net.DialTCP("tcp", nil, rAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + transceiver := NewNettyTransceiver(conn) + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + bb := new(bytes.Buffer) + + requestor.write_call_requestHeader("append", bb) + + refHeader := []byte("\x02\x00\x00\x0c\x61\x70\x70\x65\x6e\x64") + bytes := bb.Bytes() + if !reflect.DeepEqual(refHeader, bytes) { + t.Fatalf("Request_Header not equals to ref %n%x, %n%x", len(refHeader), refHeader, len(bytes), bytes) + } +} + + diff --git a/schema.go b/schema.go index c221cd4..1a45138 100644 --- a/schema.go +++ b/schema.go @@ -9,12 +9,11 @@ const handshakeRequestshema = ` {"name": "clientHash", "type": {"type": "fixed", "name": "MD5", "size": 16}}, {"name": "clientProtocol", "type": ["null", "string"]}, - {"name": "serverHash", "type": "MD5"}, - {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]} + {"name": "serverHash", "type": "MD5"} ] } ` - +//, {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]} const handshakeResponseshema = ` { "type": "record", From f38e288438438086f1522da7fcf3342840deb896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Wed, 24 Aug 2016 10:18:47 +0200 Subject: [PATCH 06/15] Correct meta encoding on frame --- requestor.go | 17 +++++++---------- requestor_test.go | 4 ++-- schema.go | 5 +++-- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/requestor.go b/requestor.go index 9e17632..dcf816b 100644 --- a/requestor.go +++ b/requestor.go @@ -105,9 +105,9 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er } func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { - //local_hash :=a.local_protocol.MD5 + local_hash :=a.local_protocol.MD5 - local_hash :=[]byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65") + //local_hash :=[]byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65") remote_name := a.remote_protocol.Name remote_hash := REMOTE_HASHES[remote_name] if len(remote_hash)==0 { @@ -122,6 +122,7 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { record.Set("clientHash", local_hash) record.Set("serverHash", remote_hash) + record.Set("meta", make(map[string]interface{})) codecHandshake, err := NewCodec(handshakeRequestshema) if err != nil { return fmt.Errorf("Avro fail to get codec handshakeRequest %v",err) @@ -161,14 +162,10 @@ func (a *Requestor) write_call_requestHeader(message_name string, frame1 io.Writ // TODO request metadata (not yet implemented) request_metadata := make(map[string]interface{}) - metaBuffer := new(bytes.Buffer) - // encode metadata - if err := META_WRITER.Encode(metaBuffer, request_metadata); err != nil { - return fmt.Errorf("Encode metadata ", err) - } - longCodec.Encode(frame1, int64(metaBuffer.Len())) - frame1.Write(metaBuffer.Bytes()) - longCodec.Encode(frame1, int64(0)) + // encode metadata + if err := META_WRITER.Encode(frame1, request_metadata); err != nil { + return fmt.Errorf("Encode metadata ", err) + } stringCodec.Encode(frame1,message_name) diff --git a/requestor_test.go b/requestor_test.go index 516e741..9a6cdcd 100644 --- a/requestor_test.go +++ b/requestor_test.go @@ -29,7 +29,7 @@ func TestWrite_handshake_request(t *testing.T) { t.Logf("Handshake_request size %v %x\n",bb.Len(), bb.Bytes()) t.Logf( "Handshake_request %v\n", bb.String()) - refHandshake := []byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x00\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x02") + refHandshake := []byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x00\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x02\x00") //bytes := bb.Bytes() //if !reflect.DeepEqual(refHandshake, bytes) { // t.Fatalf("Handshake not equals to ref %n%x, %n%x", len(refHandshake), refHandshake, len(bytes), bytes) @@ -117,7 +117,7 @@ func TestWrite_call_requestHeader(t *testing.T) { requestor.write_call_requestHeader("append", bb) - refHeader := []byte("\x02\x00\x00\x0c\x61\x70\x70\x65\x6e\x64") + refHeader := []byte("\x00\x0c\x61\x70\x70\x65\x6e\x64") bytes := bb.Bytes() if !reflect.DeepEqual(refHeader, bytes) { t.Fatalf("Request_Header not equals to ref %n%x, %n%x", len(refHeader), refHeader, len(bytes), bytes) diff --git a/schema.go b/schema.go index 1a45138..c221cd4 100644 --- a/schema.go +++ b/schema.go @@ -9,11 +9,12 @@ const handshakeRequestshema = ` {"name": "clientHash", "type": {"type": "fixed", "name": "MD5", "size": 16}}, {"name": "clientProtocol", "type": ["null", "string"]}, - {"name": "serverHash", "type": "MD5"} + {"name": "serverHash", "type": "MD5"}, + {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]} ] } ` -//, {"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]} + const handshakeResponseshema = ` { "type": "record", From 9277cbd06311c03409064440ec9981aef842bddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Wed, 24 Aug 2016 11:12:33 +0200 Subject: [PATCH 07/15] solve protocol MD5 error --- protocol.go | 11 ++++++++--- protocol_test.go | 12 ++++++++---- requestor.go | 2 -- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/protocol.go b/protocol.go index 60421c1..bc98c61 100644 --- a/protocol.go +++ b/protocol.go @@ -97,9 +97,14 @@ func NewProtocol() (Protocol, error) { err = fmt.Errorf("The namespace property must be a string.") } result.Fullname = result.Namespace +"." + result.Name - hasher := md5.New() - hasher.Write([]byte(proto)) - result.MD5 = hasher.Sum(nil) + + bb, err := json.Marshal(result) + + if err!=nil { + return result, err + } + hash := md5.Sum(bb) + result.MD5 = hash[:] return result, err } diff --git a/protocol_test.go b/protocol_test.go index d74b0a0..ca8bcce 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -4,16 +4,20 @@ import ( "testing" "bytes" "encoding/json" - "crypto/md5" "reflect" ) func TestMD5(t *testing.T) { + proto, err := NewProtocol() + if err!=nil { + t.Fatal(err) + } + expected := []byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65") + if !reflect.DeepEqual(expected, proto.MD5) { + t.Errorf("MD5 not equals: %x / %x", expected, proto.MD5) + } - - - t.Logf("golang md5 hex %x", md5.Sum([]byte(`{"protocol":"AvroSourceProtocol","namespace":"org.apache.flume.source.avro","doc":"* Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing,\n * software distributed under the License is distributed on an\n * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n * KIND, either express or implied. See the License for the\n * specific language governing permissions and limitations\n * under the License.","types":[{"type":"enum","name":"Status","symbols":["OK","FAILED","UNKNOWN"]},{"type":"record","name":"AvroFlumeEvent","fields":[{"name":"headers","type":{"type":"map","values":"string"}},{"name":"body","type":"bytes"}]}],"messages":{"append":{"request":[{"name":"event","type":"AvroFlumeEvent"}],"response":"Status"},"appendBatch":{"request":[{"name":"events","type":{"type":"array","items":"AvroFlumeEvent"}}],"response":"Status"}}}`))) } diff --git a/requestor.go b/requestor.go index dcf816b..a2117fc 100644 --- a/requestor.go +++ b/requestor.go @@ -106,8 +106,6 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { local_hash :=a.local_protocol.MD5 - - //local_hash :=[]byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65") remote_name := a.remote_protocol.Name remote_hash := REMOTE_HASHES[remote_name] if len(remote_hash)==0 { From abb77fe86513645b25bef809952678ec6938fb5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Wed, 24 Aug 2016 14:58:50 +0200 Subject: [PATCH 08/15] refactor method name --- ipc_test.go | 2 +- protocol.go | 8 ++++---- protocol_test.go | 4 ++-- requestor.go | 2 +- requestor_test.go | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ipc_test.go b/ipc_test.go index 6aeb93c..1ef7691 100644 --- a/ipc_test.go +++ b/ipc_test.go @@ -21,7 +21,7 @@ func TestRequestor(t *testing.T) { t.Fatal(err) } - flumeRecord, errFlume := protocol.getNewRecord("AvroFlumeEvent") + flumeRecord, errFlume := protocol.NewRecord("AvroFlumeEvent") if errFlume != nil { t.Fatal(errFlume) } diff --git a/protocol.go b/protocol.go index bc98c61..a97312b 100644 --- a/protocol.go +++ b/protocol.go @@ -118,20 +118,20 @@ func (p *Protocol) Json() (string, error) { return string(bb), nil } -func (p *Protocol) getMessageRequestCodec(messageName string) (Codec, error) { - json, err := p.getMessageRequestJson(messageName) +func (p *Protocol) MessageRequestCodec(messageName string) (Codec, error) { + json, err := p.MessageRequestJson(messageName) if err!= nil { return nil, err } return NewCodec(json) } -func (p *Protocol) getMessageRequestJson(messageName string) (string, error) { +func (p *Protocol) MessageRequestJson(messageName string) (string, error) { field := p.Messages[messageName].Request[0] avroType := TYPES_CACHE[field.TypeX.ref] json, err := json.Marshal(avroType) return string(json), err } -func (p *Protocol) getNewRecord(typeName string) (*Record, error) { +func (p *Protocol) NewRecord(typeName string) (*Record, error) { avroType := TYPES_CACHE[typeName] json, err := json.Marshal(avroType) if err!= nil { diff --git a/protocol_test.go b/protocol_test.go index ca8bcce..7572862 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -93,7 +93,7 @@ func TestGetCodec(t *testing.T) { } - flumeRecord, errFlume := protocol.getNewRecord("AvroFlumeEvent") + flumeRecord, errFlume := protocol.NewRecord("AvroFlumeEvent") if errFlume != nil { t.Fatal(errFlume) } @@ -103,7 +103,7 @@ func TestGetCodec(t *testing.T) { flumeRecord.Set("headers", headers) flumeRecord.Set("body", []byte("test")) bb := new(bytes.Buffer) - codec, err := protocol.getMessageRequestCodec("append") + codec, err := protocol.MessageRequestCodec("append") if err != nil { t.Fatal(err) } diff --git a/requestor.go b/requestor.go index a2117fc..994086e 100644 --- a/requestor.go +++ b/requestor.go @@ -142,7 +142,7 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { } func (a *Requestor) write_call_request(message_name string, request_datum interface{}, frame io.Writer) (err error) { - codec, err := a.local_protocol.getMessageRequestCodec(message_name) + codec, err := a.local_protocol.MessageRequestCodec(message_name) if err != nil { return fmt.Errorf("fail to get codec for message %s: %v", message_name, err) diff --git a/requestor_test.go b/requestor_test.go index 9a6cdcd..42f827c 100644 --- a/requestor_test.go +++ b/requestor_test.go @@ -66,7 +66,7 @@ func TestWrite_call_request(t *testing.T) { requestor := NewRequestor(protocol, transceiver) bb := new(bytes.Buffer) - datum, err := protocol.getNewRecord("AvroFlumeEvent") + datum, err := protocol.NewRecord("AvroFlumeEvent") if err != nil { t.Fatal(err) } @@ -83,7 +83,7 @@ func TestWrite_call_request(t *testing.T) { t.Logf("\nCall_request size %v %v\n",bb.Len(), bb.Bytes()) t.Logf("\nCall_request %v\n", bb.String()) - codec, err := protocol.getMessageRequestCodec("append") + codec, err := protocol.MessageRequestCodec("append") if err != nil { t.Fatal(err) } From 4d3728f359c1d43821d695d8f5f7262de313caac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Wed, 24 Aug 2016 15:49:51 +0200 Subject: [PATCH 09/15] add log error message on NettyTransceiver --- Transceiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Transceiver.go b/Transceiver.go index fa9296f..5ccc88e 100644 --- a/Transceiver.go +++ b/Transceiver.go @@ -40,7 +40,7 @@ func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]byte, error){ // Send request _, err := t.sock.Write(nettyFrame.Bytes()) if err!=nil { - return nil, fmt.Errorf("Fail to write on socket") + return nil, fmt.Errorf("Fail to write on socket %v", err) } //sfmt.Fprintf(os.Stdout, "BufferSize %v", nettyFrame) // Read Response From 74f9d04f14046c4d0606715697dd8a96159abfc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Thu, 25 Aug 2016 12:36:59 +0200 Subject: [PATCH 10/15] add read Handshake Response --- Transceiver.go | 37 +++++++++++++++++++++++++++++----- Transceiver_test.go | 23 ++++++++++++++++++++++ ipc_test.go | 6 ++++++ requestor.go | 36 ++++++++++++++++++++++------------ requestor_test.go | 48 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/Transceiver.go b/Transceiver.go index 5ccc88e..fb847ca 100644 --- a/Transceiver.go +++ b/Transceiver.go @@ -5,10 +5,12 @@ import ( "net" "encoding/binary" "fmt" + "os" + "io" ) type Transceiver interface { - Transceive(request []bytes.Buffer) ([]byte, error) + Transceive(request []bytes.Buffer) ([]io.Reader, error) RemoteName() string SetRemoteName(string) } @@ -33,7 +35,7 @@ func (t NettyTransceiver) SetRemoteName(name string) { t.remoteName = name } -func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]byte, error){ +func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ nettyFrame := new(bytes.Buffer) t.Pack(nettyFrame, requests) @@ -42,11 +44,14 @@ func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]byte, error){ if err!=nil { return nil, fmt.Errorf("Fail to write on socket %v", err) } - //sfmt.Fprintf(os.Stdout, "BufferSize %v", nettyFrame) + // Read Response bodyBytes := make([]byte, 1024) - t.sock.Read(bodyBytes) - return bodyBytes, nil + _, err = t.sock.Read(bodyBytes) + if err!=nil { + return nil, fmt.Errorf("Fail to read on socket %v", err) + } + return t.Unpack(bodyBytes) } func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { @@ -67,4 +72,26 @@ func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { frame.Write(requestSize) frame.Write(request.Bytes()) } +} + +func (t *NettyTransceiver) Unpack(frame []byte) ([]io.Reader, error) { + + nettyNumberFame := binary.BigEndian.Uint32(frame[4:8]) + result := make([]io.Reader, nettyNumberFame) + startFrame := uint32(8) + i:=uint32(0) + for i < nettyNumberFame { + + + messageSize := uint32(binary.BigEndian.Uint32(frame[startFrame:startFrame+4])) + fmt.Fprintf(os.Stdout, "\nnettyNumberFrame %v %v ", startFrame, frame[startFrame:startFrame+4]) + message := frame[startFrame+4:startFrame+4+messageSize] + fmt.Fprintf(os.Stdout, "\nmessage: %v", message) + startFrame = startFrame+4+messageSize + br := bytes.NewReader(message) + result[i] = br + i++ + } + + return result, nil } \ No newline at end of file diff --git a/Transceiver_test.go b/Transceiver_test.go index 71e6e2b..cdb4240 100644 --- a/Transceiver_test.go +++ b/Transceiver_test.go @@ -5,6 +5,7 @@ import ( "bytes" "reflect" + "io/ioutil" ) func TestPack(t *testing.T) { @@ -26,3 +27,25 @@ func TestPack(t *testing.T) { t.Fatalf("Frame not equals to %x: %x / %x",expectedSize, frame.Len(), frame.Bytes()) } } + +func TestUnpack(t *testing.T) { + transceiver := new(NettyTransceiver) + frame := []byte("\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x01\x0a\x00\x00\x00\x01\x0b") + respons, err := transceiver.Unpack(frame) + if err != nil { + t.Fatalf("%v",err) + } + + if len(respons)!=2 { + t.Fatalf("Number of reponse frame not equals %x / %x",2, len(respons)) + } + + var resp1 []byte + var resp2 []byte + resp1, _ =ioutil.ReadAll(respons[0]) + respons[1].Read(resp2) + if !reflect.DeepEqual(resp1, []byte("\x0a")) && !reflect.DeepEqual(resp2, []byte("\x0b")) { + t.Fatalf("Reponse message not equals (0) %x/%x; (1) %x/%x","\x0a", respons[0], "\x0b", respons[1]) + } + +} \ No newline at end of file diff --git a/ipc_test.go b/ipc_test.go index 1ef7691..a1e1b29 100644 --- a/ipc_test.go +++ b/ipc_test.go @@ -35,5 +35,11 @@ func TestRequestor(t *testing.T) { if err != nil { t.Fatal("Request: ", err) } + + err = requestor.Request("append", flumeRecord) + + if err != nil { + t.Fatal("Request: ", err) + } } diff --git a/requestor.go b/requestor.go index 994086e..96c025b 100644 --- a/requestor.go +++ b/requestor.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "log" + "os" ) var REMOTE_HASHES map[string][]byte @@ -25,6 +25,7 @@ type Requestor struct { remote_protocol Protocol remote_hash []byte send_protocol bool + send_handshake bool } func init() { var err error @@ -49,7 +50,8 @@ func NewRequestor(localProto Protocol, transceiver Transceiver) *Requestor { transceiver: transceiver, // remote_protocol: nil, // remote_hash: nil, -// send_protocol: nil, + send_protocol: false, + send_handshake: true, } } @@ -86,25 +88,32 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er // sen the handshake and call request; block until call response buffer_writers := []bytes.Buffer{*frame1, *frame2} - decoder, err := a.transceiver.Transceive(buffer_writers) + responses, err := a.transceiver.Transceive(buffer_writers) + if err!=nil { return err } - buffer_decoder := bytes.NewBuffer(decoder) + //buffer_decoder := bytes.NewBuffer(decoder) // process the handshake and call response - //ok, err := a.read_handshake_response(buffer_decoder) - fmt.Sprintf("Response %v", buffer_decoder) - //if err!=nil { - // return err - //} else if ok { + fmt.Fprintf(os.Stdout, "\nresponsee %#v", responses) + ok, err := a.read_handshake_response(responses[0]) + if err!=nil { + return err + } + a.send_handshake= !ok + + if ok { // a.read_call_response(message_name, buffer_decoder) //} else { // a.Request(message_name, request_datum) - //} + } return nil } func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { + if !a.send_handshake { + return nil + } local_hash :=a.local_protocol.MD5 remote_name := a.remote_protocol.Name remote_hash := REMOTE_HASHES[remote_name] @@ -180,11 +189,14 @@ func (a *Requestor) write_request(request_codec Codec, request_datum interface{} } func (a *Requestor) read_handshake_response(decoder io.Reader) (bool, error) { - resp, _ := ioutil.ReadAll(decoder) + if !a.send_handshake { + return true, nil + } + datum, err := HANDSHAKE_REQUESTOR_READER.Decode(decoder) if err != nil { - return false,fmt.Errorf("Fail to decode %v with error %v", resp, err) + return false,fmt.Errorf("Fail to decode %v with error %v", decoder, err) } record, ok := datum.(*Record) diff --git a/requestor_test.go b/requestor_test.go index 42f827c..2df1726 100644 --- a/requestor_test.go +++ b/requestor_test.go @@ -48,6 +48,54 @@ func TestWrite_handshake_request(t *testing.T) { } +func TestRead_handshake_reponse(t *testing.T) { + codecHandshake, err := NewCodec(handshakeResponseshema) + if err != nil { + t.Fatal(err) + } + record, err := NewRecord(RecordSchema(handshakeResponseshema)) + if err != nil { + t.Fatal(err) + } + record.Set("match", Enum{"match","BOTH"}) + record.Set("serverProtocol", nil) + record.Set("serverHash", nil) + record.Set("meta", nil) + + bb := new(bytes.Buffer) + err = codecHandshake.Encode(bb, record) + if err != nil { + t.Fatal(err) + } + t.Logf("Encode HandshakeResponse %v", bb.Bytes()) + + + _, err = codecHandshake.Decode(bytes.NewReader(bb.Bytes())) + if err != nil { + t.Fatal(err) + } + + rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") + conn, err := net.DialTCP("tcp", nil, rAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + transceiver := NewNettyTransceiver(conn) + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + _, err = requestor.read_handshake_response(bytes.NewReader(bb.Bytes())) + if err != nil { + t.Fatal(err) + } + +} + func TestWrite_call_request(t *testing.T) { //t.SkipNow() From 185b8a401dd6f1b7d64f7354c38deb3a539b1d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Fri, 26 Aug 2016 12:04:10 +0200 Subject: [PATCH 11/15] end of dev --- Transceiver.go | 6 - ipc_test.go => examples/flume/client.go | 24 +-- ipc.go | 264 ------------------------ protocol.go | 14 ++ protocol_test.go | 2 +- requestor.go | 53 ++++- requestor_test.go | 41 ++++ 7 files changed, 115 insertions(+), 289 deletions(-) rename ipc_test.go => examples/flume/client.go (72%) delete mode 100644 ipc.go diff --git a/Transceiver.go b/Transceiver.go index fb847ca..38d73b5 100644 --- a/Transceiver.go +++ b/Transceiver.go @@ -5,7 +5,6 @@ import ( "net" "encoding/binary" "fmt" - "os" "io" ) @@ -75,18 +74,13 @@ func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { } func (t *NettyTransceiver) Unpack(frame []byte) ([]io.Reader, error) { - nettyNumberFame := binary.BigEndian.Uint32(frame[4:8]) result := make([]io.Reader, nettyNumberFame) startFrame := uint32(8) i:=uint32(0) for i < nettyNumberFame { - - messageSize := uint32(binary.BigEndian.Uint32(frame[startFrame:startFrame+4])) - fmt.Fprintf(os.Stdout, "\nnettyNumberFrame %v %v ", startFrame, frame[startFrame:startFrame+4]) message := frame[startFrame+4:startFrame+4+messageSize] - fmt.Fprintf(os.Stdout, "\nmessage: %v", message) startFrame = startFrame+4+messageSize br := bytes.NewReader(message) result[i] = br diff --git a/ipc_test.go b/examples/flume/client.go similarity index 72% rename from ipc_test.go rename to examples/flume/client.go index a1e1b29..8093a68 100644 --- a/ipc_test.go +++ b/examples/flume/client.go @@ -1,45 +1,45 @@ -package goavro - +package main import ( - "testing" + "github.com/sebglon/goavro" "net" + "log" ) -func TestRequestor(t *testing.T) { +func main() { //t.SkipNow() rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") conn, err := net.DialTCP("tcp", nil, rAddr) if err != nil { - t.Fatal(err) + log.Fatal(err) } defer conn.Close() - transceiver := NewNettyTransceiver(conn) - protocol, err := NewProtocol() + transceiver := goavro.NewNettyTransceiver(conn) + protocol, err := goavro.NewProtocol() if err != nil { - t.Fatal(err) + log.Fatal(err) } flumeRecord, errFlume := protocol.NewRecord("AvroFlumeEvent") if errFlume != nil { - t.Fatal(errFlume) + log.Fatal(errFlume) } headers := make(map[string]interface{}) headers["host_header"] = "127.0.0.1" flumeRecord.Set("headers", headers) flumeRecord.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) - requestor := NewRequestor(protocol, transceiver) + requestor := goavro.NewRequestor(protocol, transceiver) err = requestor.Request("append", flumeRecord) if err != nil { - t.Fatal("Request: ", err) + log.Fatal("Request: ", err) } err = requestor.Request("append", flumeRecord) if err != nil { - t.Fatal("Request: ", err) + log.Fatal("Request: ", err) } } diff --git a/ipc.go b/ipc.go deleted file mode 100644 index 18f2e7e..0000000 --- a/ipc.go +++ /dev/null @@ -1,264 +0,0 @@ -// Package avro provides th log driver for forwarding server logs to -// flume endpoints. -package goavro - -import ( - "fmt" - "net/rpc" - "bytes" - "io" -) -type Client struct { - hostname string - extra map[string]interface{} - conn *rpc.Client - codecFlume Codec - codecDocker Codec - codecMeta Codec - codecString Codec - protocol Protocol -} - -const ( - defaultHost = "10.98.80.113" - defaultPort = "63001" - - hostKey = "avro-host" - portKey = "avro-port" -) - -const AVRO_SCHEMA_LITERAL_HEADER = "flume.avro.schema.literal" - - -const stringSchema = ` -{"type": "string"} -` -const flumeSchema = ` -{ - "type": "record", - "name": "AvroFlumeEvent", - "fields": [{ - "name": "headers", - "type": { - "type": "map", - "values": "string" - } - }, { - "name": "body", - "type": "bytes" - }] -} -` - -const recordSchema = ` -{ - "type": "record", - "name": "docker_logs", - "doc:": "A basic schema for storing docker container logs", - "namespace": "docker", - "fields": [ - { - "doc": "Docker container ID", - "type": "string", - "name": "container_id" - }, - { - "doc": "Docker container Name", - "type": "string", - "name": "container_name" - }, - { - "doc": "Docker image ID", - "type": "string", - "name": "image_id" - }, - { - "doc": "Docker image Name", - "type": "string", - "name": "image_name" - }, - { - "doc": "Docker container commmand", - "type": "string", - "name": "command" - }, - { - "doc": "Docker container created timestamp", - "type": "long", - "name": "created" - }, - { - "doc": "Source of log (stdout, stderr)", - "type": "string", - "name": "source" - }, - { - "doc": "Docker host", - "type": "string", - "name": "docker_host" - }, - { - "doc": "Log message", - "type": "string", - "name": "log" - }, - { - "doc": "Unix timestamp in milliseconds", - "type": "long", - "name": "timestamp" - } - ] -} -` - -func NewDefaultClient() (*Client, error) { - return NewClient(defaultHost, defaultPort) - -} -// New create a avro logger using the configuration passed in on -// the context. -func NewClient(host string, port string) (*Client, error) { - servAddr := host+":"+port - conn, err := rpc.Dial("tcp", servAddr) - if err != nil { - return nil, err - } - - - codec, err := NewCodec(flumeSchema) - if err != nil { - return nil, err - } - - codecDocker, err := NewCodec(recordSchema) - if err != nil { - return nil, err - } - - codecMeta, err := NewCodec(metadataSchema) - if err != nil { - return nil, err - } - - codecString, err := NewCodec(stringSchema) - if err != nil { - return nil, err - } - - proto, err := NewProtocol() - if err != nil { - return nil, err - } - - return &Client { - extra: make(map[string]interface{}), - hostname: "TODO", - conn: conn, - codecFlume: codec, - codecDocker: codecDocker, - codecMeta: codecMeta, - codecString: codecString, - protocol: proto, - }, nil -} - -func (a *Client) Write_handshake_request( buffer io.Writer ) (err error) { - local_hash :=a.protocol.MD5 - // remote_name :="" // only setted by handshake response - remote_hash := make([]byte,0) - if len(remote_hash)==0 { - remote_hash = local_hash - } - - record, err := NewRecord(RecordSchema(handshakeRequestshema)) - if err != nil { - return fmt.Errorf("Avro fail to init record handshakeRequest",err) - } - - record.Set("clientHash", local_hash) - record.Set("serverHash", remote_hash) -// record.Set("clientProtocol", a.protocol.Name) - codecHandshake, err := NewCodec(handshakeRequestshema) - if err != nil { - return err - } - - if err = codecHandshake.Encode(buffer, record); err !=nil { - return fmt.Errorf("Encode handshakeRequest ",err) - } - return nil -} - - -func (a *Client) Log(msg string) error { - - bb := new(bytes.Buffer) - a.Write_handshake_request(bb) - - flumeRecord, errFlume := NewRecord(RecordSchema(flumeSchema)) - if errFlume != nil { - return fmt.Errorf("Avro fail to init record",errFlume) - } - headers := make(map[string]interface{}) - headers[AVRO_SCHEMA_LITERAL_HEADER] = stringSchema - headers["host_header"] = "127.0.0.1" - flumeRecord.Set("headers", headers) - flumeRecord.Set("body", []byte("test")) - - - // encode metadata - if err := a.codecMeta.Encode(bb, make(map[string]interface{})); err !=nil { - return fmt.Errorf("Encode metadata ",err) - } - - // encode message name - if err := a.codecString.Encode(bb, "append"); err !=nil { - return fmt.Errorf("Encode message name ",err) - } - - - // encode message parameters - - if err := a.codecFlume.Encode(bb, flumeRecord); err !=nil { - return fmt.Errorf("Encode flumeRecord ",err) - } - - return a.Send(bb.Bytes()) -} - -func (a *Client) Send(bytes []byte ) error { - var reply []byte - err := a.conn.Call("", bytes,&reply) - if err !=nil { - return err - } - return nil -} - -func (a *Client) Close() error { - return a.conn.Close() -} - - -// ValidateLogOpt looks for avro specific log option avro-host avro-port. -func ValidateLogOpt(cfg map[string]string) error { - for key := range cfg { - switch key { - case "env": - case "labes": - case hostKey: - case portKey: - // Accepted - default: - return fmt.Errorf("unknown log opt '%s' for avro log driver", key) - } - } - if len(cfg[hostKey]) == 0 { - cfg[hostKey] = defaultHost - } - if len(cfg[portKey]) == 0 { - cfg[portKey] = defaultPort - } - return nil -} - diff --git a/protocol.go b/protocol.go index a97312b..df62a46 100644 --- a/protocol.go +++ b/protocol.go @@ -118,6 +118,20 @@ func (p *Protocol) Json() (string, error) { return string(bb), nil } + +func (p *Protocol) MessageResponseCodec(messageName string) (Codec, error) { + json, err := p.MessageResponseJson(messageName) + if err!= nil { + return nil, err + } + return NewCodec(json) +} +func (p *Protocol) MessageResponseJson(messageName string) (string, error) { + field := p.Messages[messageName].Response + avroType := TYPES_CACHE[field] + json, err := json.Marshal(avroType) + return string(json), err +} func (p *Protocol) MessageRequestCodec(messageName string) (Codec, error) { json, err := p.MessageRequestJson(messageName) if err!= nil { diff --git a/protocol_test.go b/protocol_test.go index 7572862..3b677ef 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -82,7 +82,7 @@ func TestToJson(t *testing.T) { t.Fatal("%#v", err) } if result!= jsonCompact(proto) { - t.Errorf("Proto to Json not equals; Expected %#v, actual %#v",jsonCompact(proto), result) + t.Errorf("Proto to Json not equals; Expected \n%#v\nactual \n%#v",jsonCompact(proto), result) } } diff --git a/requestor.go b/requestor.go index 96c025b..7ac652d 100644 --- a/requestor.go +++ b/requestor.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "log" - "os" ) var REMOTE_HASHES map[string][]byte @@ -95,7 +94,6 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er } //buffer_decoder := bytes.NewBuffer(decoder) // process the handshake and call response - fmt.Fprintf(os.Stdout, "\nresponsee %#v", responses) ok, err := a.read_handshake_response(responses[0]) if err!=nil { return err @@ -103,8 +101,10 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er a.send_handshake= !ok if ok { - // a.read_call_response(message_name, buffer_decoder) - //} else { + a.read_call_responseCode(responses[1]) + if err!=nil { + return err + } // a.Request(message_name, request_datum) } return nil @@ -253,7 +253,7 @@ func (a *Requestor) read_handshake_response(decoder io.Reader) (bool, error) { return we_have_matching_schema, nil } -func (a *Requestor) read_call_response(message_name string, decoder io.Writer) { +func (a *Requestor) read_call_responseCode(decoder io.Reader) error { // The format of a call response is: // * response metadata, a map with values of type bytes // * a one-byte error flag boolean, followed by either: @@ -261,8 +261,49 @@ func (a *Requestor) read_call_response(message_name string, decoder io.Writer) { // the message response, serialized per the message's response schema. // * if the error flag is true, // the error, serialized per the message's error union schema. -// META_READER.Decode(decoder) + _, err := META_READER.Decode(decoder) + + if err != nil { + return fmt.Errorf("Decode metadata ", err) + } + return nil + } +func (a *Requestor) read_call_responseMessage(message_name string, decoder io.Reader ) error { + codec, err := a.local_protocol.MessageResponseCodec(message_name) + + if err != nil { + return fmt.Errorf("fail to get response codec for message %s: %v", message_name, err) + } + + datum, err := codec.Decode(decoder); + if err != nil { + + return fmt.Errorf("Fail to decode %v with error %v", decoder, err) + } + status, ok := datum.(string) + if !ok { + return fmt.Errorf("Fail to decode Status response %v", datum) + } + + switch status { + case "OK": + err = nil + case "FAILED": + err = fmt.Errorf("Reponse failure. status == %v", status) + + case "UNKNOWN": + err = fmt.Errorf("Reponse failure. match == %v", status) + + default: + err = fmt.Errorf("Unexpected status: %v", status) + } + + return err + + +} + diff --git a/requestor_test.go b/requestor_test.go index 2df1726..36b2da3 100644 --- a/requestor_test.go +++ b/requestor_test.go @@ -172,4 +172,45 @@ func TestWrite_call_requestHeader(t *testing.T) { } } +func TestRead_call_responseMessage(t *testing.T) { + //t.SkipNow() + + rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") + conn, err := net.DialTCP("tcp", nil, rAddr) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + transceiver := NewNettyTransceiver(conn) + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + + codec, err := protocol.MessageResponseCodec("append") + if err != nil { + t.Fatal(err) + } + bb := new(bytes.Buffer) + codec.Encode(bb, Enum{"Status", "OK"}) + t.Logf("Bytes for OK %x",bb.Bytes() ) + + + err = requestor.read_call_responseMessage("append", bb) + if err != nil { + t.Fatal(err) + } + + codec.Encode(bb, Enum{"Status", "FAILED"}) + t.Logf("Bytes for FAILED %x",bb.Bytes() ) + err = requestor.read_call_responseMessage("append", bb) + if err == nil || err.Error() != "Reponse failure. status == FAILED"{ + t.Fatalf("Status FAILED can return error") + } + +} + From 002ce835981771506d9fbc6467fe12a7b38f2d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Tue, 30 Aug 2016 11:15:59 +0200 Subject: [PATCH 12/15] Add asyncConnect and TimeOut to netty Transceiver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sébastien GLON --- Transceiver.go | 91 ------- examples/flume/client.go | 14 +- protocol_test.go | 1 - requestor.go | 30 ++- requestor_test.go | 52 ++-- transceiver/Tranceiver.go | 11 + transceiver/netty/Netty.go | 251 ++++++++++++++++++ .../netty/Netty_test.go | 74 +++++- 8 files changed, 383 insertions(+), 141 deletions(-) delete mode 100644 Transceiver.go create mode 100644 transceiver/Tranceiver.go create mode 100644 transceiver/netty/Netty.go rename Transceiver_test.go => transceiver/netty/Netty_test.go (50%) diff --git a/Transceiver.go b/Transceiver.go deleted file mode 100644 index 38d73b5..0000000 --- a/Transceiver.go +++ /dev/null @@ -1,91 +0,0 @@ -package goavro - -import ( - "bytes" - "net" - "encoding/binary" - "fmt" - "io" -) - -type Transceiver interface { - Transceive(request []bytes.Buffer) ([]io.Reader, error) - RemoteName() string - SetRemoteName(string) -} - - -type NettyTransceiver struct { - sock *net.TCPConn - remoteName string -} -func NewNettyTransceiver(sock *net.TCPConn) Transceiver{ - return NettyTransceiver { - sock: sock, - } -} -func (t NettyTransceiver) RemoteName() string { - return t.remoteName - return t.remoteName - return t.remoteName -} - -func (t NettyTransceiver) SetRemoteName(name string) { - t.remoteName = name -} - -func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ - nettyFrame := new(bytes.Buffer) - t.Pack(nettyFrame, requests) - - // Send request - _, err := t.sock.Write(nettyFrame.Bytes()) - if err!=nil { - return nil, fmt.Errorf("Fail to write on socket %v", err) - } - - // Read Response - bodyBytes := make([]byte, 1024) - _, err = t.sock.Read(bodyBytes) - if err!=nil { - return nil, fmt.Errorf("Fail to read on socket %v", err) - } - return t.Unpack(bodyBytes) -} - -func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { - // Set Netty Serial - - nettySerial :=make([]byte, 4) - binary.BigEndian.PutUint32(nettySerial, uint32(1)) - frame.Write(nettySerial) - - - nettySizeBuffer :=make([]byte, 4) - binary.BigEndian.PutUint32(nettySizeBuffer, uint32(len(requests))) - frame.Write(nettySizeBuffer) - - for _, request := range requests { - requestSize :=make([]byte, 4) - binary.BigEndian.PutUint32(requestSize, uint32(request.Len())) - frame.Write(requestSize) - frame.Write(request.Bytes()) - } -} - -func (t *NettyTransceiver) Unpack(frame []byte) ([]io.Reader, error) { - nettyNumberFame := binary.BigEndian.Uint32(frame[4:8]) - result := make([]io.Reader, nettyNumberFame) - startFrame := uint32(8) - i:=uint32(0) - for i < nettyNumberFame { - messageSize := uint32(binary.BigEndian.Uint32(frame[startFrame:startFrame+4])) - message := frame[startFrame+4:startFrame+4+messageSize] - startFrame = startFrame+4+messageSize - br := bytes.NewReader(message) - result[i] = br - i++ - } - - return result, nil -} \ No newline at end of file diff --git a/examples/flume/client.go b/examples/flume/client.go index 8093a68..b687292 100644 --- a/examples/flume/client.go +++ b/examples/flume/client.go @@ -1,21 +1,17 @@ package main import ( "github.com/sebglon/goavro" - "net" "log" - + "github.com/sebglon/goavro/transceiver/netty" + "time" ) func main() { //t.SkipNow() - rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") - conn, err := net.DialTCP("tcp", nil, rAddr) + transceiver,err := netty.NewTransceiver(netty.Config{AsyncConnect:true, NettyHost:"10.98.80.113"}) if err != nil { log.Fatal(err) } - defer conn.Close() - - transceiver := goavro.NewNettyTransceiver(conn) protocol, err := goavro.NewProtocol() if err != nil { log.Fatal(err) @@ -30,12 +26,14 @@ func main() { flumeRecord.Set("headers", headers) flumeRecord.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) requestor := goavro.NewRequestor(protocol, transceiver) + + err = requestor.Request("append", flumeRecord) if err != nil { log.Fatal("Request: ", err) } - + time.Sleep(5 * time.Second) err = requestor.Request("append", flumeRecord) if err != nil { diff --git a/protocol_test.go b/protocol_test.go index 3b677ef..13f6b8a 100644 --- a/protocol_test.go +++ b/protocol_test.go @@ -98,7 +98,6 @@ func TestGetCodec(t *testing.T) { t.Fatal(errFlume) } headers := make(map[string]interface{}) - headers[AVRO_SCHEMA_LITERAL_HEADER] = stringSchema headers["host_header"] = "127.0.0.1" flumeRecord.Set("headers", headers) flumeRecord.Set("body", []byte("test")) diff --git a/requestor.go b/requestor.go index 7ac652d..9359882 100644 --- a/requestor.go +++ b/requestor.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log" + "github.com/sebglon/goavro/transceiver" ) var REMOTE_HASHES map[string][]byte @@ -20,7 +21,7 @@ var HANDSHAKE_REQUESTOR_READER Codec type Requestor struct { // Base class for the client side of protocol interaction. local_protocol Protocol - transceiver Transceiver + transceiver transceiver.Transceiver remote_protocol Protocol remote_hash []byte send_protocol bool @@ -43,7 +44,7 @@ func init() { } -func NewRequestor(localProto Protocol, transceiver Transceiver) *Requestor { +func NewRequestor(localProto Protocol, transceiver transceiver.Transceiver) *Requestor { return &Requestor{ local_protocol: localProto, transceiver: transceiver, @@ -57,12 +58,12 @@ func NewRequestor(localProto Protocol, transceiver Transceiver) *Requestor { func (a *Requestor) RemoteProtocol(proto Protocol) { a.remote_protocol = proto - REMOTE_PROTOCOLS[a.transceiver.RemoteName()] = proto + REMOTE_PROTOCOLS[proto.Name] = proto } func (a *Requestor) RemoteHash(hash []byte) { a.remote_hash = hash - REMOTE_HASHES[a.transceiver.RemoteName()] = hash + REMOTE_HASHES[a.remote_protocol.Name] = hash } func (a *Requestor) Request(message_name string, request_datum interface{}) error { @@ -94,18 +95,21 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er } //buffer_decoder := bytes.NewBuffer(decoder) // process the handshake and call response - ok, err := a.read_handshake_response(responses[0]) - if err!=nil { - return err - } - a.send_handshake= !ok - if ok { - a.read_call_responseCode(responses[1]) - if err!=nil { + if len(responses) >0 { + ok, err := a.read_handshake_response(responses[0]) + if err != nil { return err } - // a.Request(message_name, request_datum) + a.send_handshake = !ok + + if ok { + a.read_call_responseCode(responses[1]) + if err != nil { + return err + } + // a.Request(message_name, request_datum) + } } return nil } diff --git a/requestor_test.go b/requestor_test.go index 36b2da3..98beac3 100644 --- a/requestor_test.go +++ b/requestor_test.go @@ -5,6 +5,7 @@ import ( "net" "bytes" "reflect" + netty "github.com/sebglon/goavro/transceiver/netty" ) func TestWrite_handshake_request(t *testing.T) { @@ -16,7 +17,10 @@ func TestWrite_handshake_request(t *testing.T) { } defer conn.Close() - transceiver := NewNettyTransceiver(conn) + transceiver, err := netty.NewTransceiver(netty.Config{}) + if err != nil { + t.Fatal(err) + } protocol, err := NewProtocol() if err != nil { t.Fatal(err) @@ -75,14 +79,12 @@ func TestRead_handshake_reponse(t *testing.T) { t.Fatal(err) } - rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") - conn, err := net.DialTCP("tcp", nil, rAddr) + + transceiver, err := netty.NewTransceiver(netty.Config{}) if err != nil { t.Fatal(err) } - defer conn.Close() - transceiver := NewNettyTransceiver(conn) protocol, err := NewProtocol() if err != nil { t.Fatal(err) @@ -96,17 +98,21 @@ func TestRead_handshake_reponse(t *testing.T) { } +type Conn struct { + bytes.Buffer +} +func (c *Conn) Close() error { + return nil +} func TestWrite_call_request(t *testing.T) { //t.SkipNow() - rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") - conn, err := net.DialTCP("tcp", nil, rAddr) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - transceiver := NewNettyTransceiver(conn) + + transceiver, err := netty.NewTransceiver(netty.Config{}) + buf := &Conn{} + transceiver.Conn = buf + protocol, err := NewProtocol() if err != nil { t.Fatal(err) @@ -147,14 +153,10 @@ func TestWrite_call_request(t *testing.T) { func TestWrite_call_requestHeader(t *testing.T) { //t.SkipNow() - rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") - conn, err := net.DialTCP("tcp", nil, rAddr) - if err != nil { - t.Fatal(err) - } - defer conn.Close() + transceiver, err := netty.NewTransceiver(netty.Config{}) + buf := &Conn{} + transceiver.Conn = buf - transceiver := NewNettyTransceiver(conn) protocol, err := NewProtocol() if err != nil { t.Fatal(err) @@ -173,16 +175,12 @@ func TestWrite_call_requestHeader(t *testing.T) { } func TestRead_call_responseMessage(t *testing.T) { - //t.SkipNow() - rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") - conn, err := net.DialTCP("tcp", nil, rAddr) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - transceiver := NewNettyTransceiver(conn) + transceiver, err := netty.NewTransceiver(netty.Config{}) + buf := &Conn{} + transceiver.Conn = buf + protocol, err := NewProtocol() if err != nil { t.Fatal(err) diff --git a/transceiver/Tranceiver.go b/transceiver/Tranceiver.go new file mode 100644 index 0000000..bf2dffb --- /dev/null +++ b/transceiver/Tranceiver.go @@ -0,0 +1,11 @@ +package transceiver + +import ( + "bytes" + "io" +) + +type Transceiver interface { + Transceive(request []bytes.Buffer) ([]io.Reader, error) + +} \ No newline at end of file diff --git a/transceiver/netty/Netty.go b/transceiver/netty/Netty.go new file mode 100644 index 0000000..9b66ac4 --- /dev/null +++ b/transceiver/netty/Netty.go @@ -0,0 +1,251 @@ +package netty + +import ( + "bytes" + "net" + "encoding/binary" + "fmt" + "io" + "sync" + "time" + "strconv" + "math" + "os" +) + + + +const ( + defaultHost = "127.0.0.1" + defaultNetwork = "tcp" + defaultSocketPath = "" + defaultPort = 63001 + defaultTimeout = 3 * time.Second + defaultBufferLimit = 8 * 1024 * 1024 + defaultRetryWait = 500 + defaultMaxRetry = 13 + defaultReconnectWaitIncreRate = 1.5 +) + +type Config struct { + NettyPort int `json:"netty_port"` + NettyHost string `json:"netty_host"` + NettyNetwork string `json:"netty_network"` + NettySocketPath string `json:"netty_socket_path"` + Timeout time.Duration `json:"timeout"` + AsyncConnect bool `json:"async_connect"` + BufferLimit int `json:"buffer_limit"` + RetryWait int `json:"retry_wait"` + MaxRetry int `json:"max_retry"` +} + + +type NettyTransceiver struct { + Config + Conn io.ReadWriteCloser + reconnecting bool + mu sync.Mutex + pending []byte +} +func NewTransceiver(config Config) (f* NettyTransceiver, err error){ + if config.NettyNetwork == "" { + config.NettyNetwork = defaultNetwork + } + if config.NettyHost == "" { + config.NettyHost = defaultHost + } + if config.NettyPort == 0 { + config.NettyPort = defaultPort + } + if config.NettySocketPath == "" { + config.NettySocketPath = defaultSocketPath + } + if config.Timeout == 0 { + config.Timeout = defaultTimeout + } + if config.BufferLimit == 0 { + config.BufferLimit = defaultBufferLimit + } + if config.RetryWait == 0 { + config.RetryWait = defaultRetryWait + } + if config.MaxRetry == 0 { + config.MaxRetry = defaultMaxRetry + } + if config.AsyncConnect { + f = &NettyTransceiver{Config: config, reconnecting: true} + f.reconnect() + } else { + f = &NettyTransceiver{Config: config, reconnecting: false} + err = f.connect() + } + return +} + +func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ + nettyFrame := new(bytes.Buffer) + t.Pack(nettyFrame, requests) + + // Send request + t.pending = append(t.pending, nettyFrame.Bytes()...) + if err := t.send(); err != nil { + t.close() + if len(t.pending) > t.Config.BufferLimit { + t.flushBuffer() + } + } else { + t.flushBuffer() + } + + // Read Response + bodyBytes := make([]byte, 1024) + t.receive(bodyBytes) +// if err!=nil { +// return nil, fmt.Errorf("Fail to read on socket %v", err) +// } + return t.Unpack(bodyBytes) +} + +func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { + // Set Netty Serial + + nettySerial :=make([]byte, 4) + binary.BigEndian.PutUint32(nettySerial, uint32(1)) + frame.Write(nettySerial) + + + nettySizeBuffer :=make([]byte, 4) + binary.BigEndian.PutUint32(nettySizeBuffer, uint32(len(requests))) + frame.Write(nettySizeBuffer) + + for _, request := range requests { + requestSize :=make([]byte, 4) + binary.BigEndian.PutUint32(requestSize, uint32(request.Len())) + frame.Write(requestSize) + frame.Write(request.Bytes()) + } +} + +func (t *NettyTransceiver) Unpack(frame []byte) ([]io.Reader, error) { + nettyNumberFame := binary.BigEndian.Uint32(frame[4:8]) + result := make([]io.Reader, nettyNumberFame) + startFrame := uint32(8) + i:=uint32(0) + for i < nettyNumberFame { + messageSize := uint32(binary.BigEndian.Uint32(frame[startFrame:startFrame+4])) + message := frame[startFrame+4:startFrame+4+messageSize] + startFrame = startFrame+4+messageSize + br := bytes.NewReader(message) + result[i] = br + i++ + } + + return result, nil +} + +// connect establishes a new connection using the specified transport. +func (f *NettyTransceiver) connect() (error) { + var err error + switch f.Config.NettyNetwork { + case "tcp": + f.Conn, err = net.DialTimeout(f.Config.NettyNetwork, f.Config.NettyHost+":"+strconv.Itoa(f.Config.NettyPort), f.Config.Timeout) + case "unix": + f.Conn, err = net.DialTimeout(f.Config.NettyNetwork, f.Config.NettySocketPath, f.Config.Timeout) + default: + err = net.UnknownNetworkError(f.Config.NettyNetwork) + } + return err +} + + +func e(x, y float64) int { + return int(math.Pow(x, y)) +} + +func (f *NettyTransceiver) reconnect() { + go func() { + + for i := 0; ; i++ { + + err := f.connect() + if err == nil { + f.mu.Lock() + f.reconnecting = false + f.mu.Unlock() + break + } else { + if i == f.Config.MaxRetry { + panic("Netty#reconnect: failed to reconnect!") + } + waitTime := f.Config.RetryWait * e(defaultReconnectWaitIncreRate, float64(i-1)) + time.Sleep(time.Duration(waitTime) * time.Millisecond) + } + } + }() +} + +func (f *NettyTransceiver) flushBuffer() { + f.mu.Lock() + defer f.mu.Unlock() + f.pending = f.pending[0:0] +} + + +// Close closes the connection. +func (f *NettyTransceiver) Close() (err error) { + if len(f.pending) > 0 { + err = f.send() + } + f.close() + return +} + +// close closes the connection. +func (f *NettyTransceiver) close() (err error) { + if f.Conn != nil { + f.mu.Lock() + defer f.mu.Unlock() + } else { + return + } + if f.Conn != nil { + f.Conn.Close() + f.Conn = nil + } + return +} + +func (f *NettyTransceiver) send() (err error) { + if f.Conn == nil { + if f.reconnecting == false { + f.mu.Lock() + f.reconnecting = true + f.mu.Unlock() + f.reconnect() + } + err = fmt.Errorf("Netty#send: can't send logs, client is reconnecting") + } else { + f.mu.Lock() + _, err = f.Conn.Write(f.pending) + f.mu.Unlock() + } + return +} + +func (f *NettyTransceiver) receive(resp []byte) (err error) { + + if f.Conn == nil { + if f.reconnecting == false { + f.mu.Lock() + f.reconnecting = true + f.mu.Unlock() + f.reconnect() + } + err = fmt.Errorf("Netty#receive: can't send logs, client is reconnecting") + } else { + f.mu.Lock() + _, err = f.Conn.Read(resp) + f.mu.Unlock() + } + return +} \ No newline at end of file diff --git a/Transceiver_test.go b/transceiver/netty/Netty_test.go similarity index 50% rename from Transceiver_test.go rename to transceiver/netty/Netty_test.go index cdb4240..7b8de8b 100644 --- a/Transceiver_test.go +++ b/transceiver/netty/Netty_test.go @@ -1,4 +1,4 @@ -package goavro +package netty import ( "testing" @@ -6,8 +6,80 @@ import ( "bytes" "reflect" "io/ioutil" + "runtime" + "net" ) +const ( + RECV_BUF_LEN = 1024 +) + +type Conn struct { + bytes.Buffer +} + +func (c *Conn) Close() error { + return nil +} + +func init() { + numProcs := runtime.NumCPU() + if numProcs < 2 { + numProcs = 2 + } + runtime.GOMAXPROCS(numProcs) + + listener, err := net.Listen("tcp", "0.0.0.0:6666") + if err != nil { + println("error listening:", err.Error()) + } + go func() { + for { + conn, err := listener.Accept() + if err != nil { + println("Error accept:", err.Error()) + return + } + go EchoFunc(conn) + } + }() +} + +func EchoFunc(conn net.Conn) { + for { + buf := make([]byte, RECV_BUF_LEN) + n, err := conn.Read(buf) + if err != nil { + println("Error reading:", err.Error()) + return + } + println("received ", n, " bytes of data =", string(buf)) + } +} + +func TestTransceive(t *testing.T) { + f := &NettyTransceiver{Config: Config{}, reconnecting: false} + + buf := &Conn{} + f.Conn = buf + + msg := "This is test writing." + bmsg := make([]bytes.Buffer, 1) + bmsg[0] = *bytes.NewBuffer([]byte(msg)) + + resp, err := f.Transceive(bmsg) + if err != nil { + t.Error(err) + } + + brcv := make([]byte, len([]byte(msg))) + resp[0].Read(brcv) + rcv := string(brcv) + if rcv != msg { + t.Errorf("got %s, except %s", rcv, msg) + } + +} func TestPack(t *testing.T) { transceiver := new(NettyTransceiver) frame := new(bytes.Buffer) From 435858f9c8c7558f981988435798f1ada68fd244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Tue, 6 Sep 2016 11:44:30 +0200 Subject: [PATCH 13/15] Remove async and retry; replace by connection pool --- examples/flume/client.go | 17 ++- requestor.go | 49 +++--- requestor_test.go | 261 +++++++++++++++++--------------- transceiver/Tranceiver.go | 25 ++- transceiver/connection.go | 46 ++++++ transceiver/netty/Netty.go | 250 ++++++++---------------------- transceiver/netty/Netty_test.go | 32 ++-- transceiver/pool.go | 135 +++++++++++++++++ 8 files changed, 461 insertions(+), 354 deletions(-) create mode 100644 transceiver/connection.go create mode 100644 transceiver/pool.go diff --git a/examples/flume/client.go b/examples/flume/client.go index b687292..80684c4 100644 --- a/examples/flume/client.go +++ b/examples/flume/client.go @@ -8,7 +8,7 @@ import ( func main() { //t.SkipNow() - transceiver,err := netty.NewTransceiver(netty.Config{AsyncConnect:true, NettyHost:"10.98.80.113"}) + transceiver,err := netty.NewTransceiver(netty.Config{AsyncConnect:false, NettyHost:"192.168.11.152"}) if err != nil { log.Fatal(err) } @@ -24,20 +24,27 @@ func main() { headers := make(map[string]interface{}) headers["host_header"] = "127.0.0.1" flumeRecord.Set("headers", headers) - flumeRecord.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) - requestor := goavro.NewRequestor(protocol, transceiver) + requestor := goavro.NewRequestor(protocol, transceiver) + flumeRecord.Set("body", []byte("test 1")) err = requestor.Request("append", flumeRecord) if err != nil { - log.Fatal("Request: ", err) + log.Fatal("Request 1: ", err) } + + log.Printf("Test 1 OK") + + time.Sleep(5 * time.Second) + flumeRecord.Set("body", []byte("test 2")) err = requestor.Request("append", flumeRecord) if err != nil { - log.Fatal("Request: ", err) + log.Fatal("Request 2: ", err) } + log.Printf("Test 2 OK") + } diff --git a/requestor.go b/requestor.go index 9359882..8eaefa9 100644 --- a/requestor.go +++ b/requestor.go @@ -45,7 +45,8 @@ func init() { } func NewRequestor(localProto Protocol, transceiver transceiver.Transceiver) *Requestor { - return &Requestor{ + + r := &Requestor{ local_protocol: localProto, transceiver: transceiver, // remote_protocol: nil, @@ -53,6 +54,8 @@ func NewRequestor(localProto Protocol, transceiver transceiver.Transceiver) *Req send_protocol: false, send_handshake: true, } + transceiver.InitHandshake(r.write_handshake_request, r.read_handshake_response) + return r } @@ -72,12 +75,7 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er frame1 := new(bytes.Buffer) frame2 := new(bytes.Buffer) - err := a.write_handshake_request(frame1) - if err!=nil { - return err - } - - err = a.write_call_requestHeader(message_name, frame1) + err := a.write_call_requestHeader(message_name, frame1) if err!=nil { return err } @@ -91,33 +89,24 @@ func (a *Requestor) Request(message_name string, request_datum interface{}) er responses, err := a.transceiver.Transceive(buffer_writers) if err!=nil { - return err + return fmt.Errorf("Fail to transceive %v", err) } //buffer_decoder := bytes.NewBuffer(decoder) // process the handshake and call response if len(responses) >0 { - ok, err := a.read_handshake_response(responses[0]) + a.read_call_responseCode(responses[1]) if err != nil { return err } - a.send_handshake = !ok - - if ok { - a.read_call_responseCode(responses[1]) - if err != nil { - return err - } - // a.Request(message_name, request_datum) - } + // a.Request(message_name, request_datum) } return nil } -func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { - if !a.send_handshake { - return nil - } +func (a *Requestor) write_handshake_request() (handshake []byte ,err error) { + buffer := new(bytes.Buffer) + defer buffer.Write(handshake) local_hash :=a.local_protocol.MD5 remote_name := a.remote_protocol.Name remote_hash := REMOTE_HASHES[remote_name] @@ -128,7 +117,8 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { record, err := NewRecord(RecordSchema(handshakeRequestshema)) if err != nil { - return fmt.Errorf("Avro fail to init record handshakeRequest %v",err) + err = fmt.Errorf("Avro fail to init record handshakeRequest %v",err) + return } record.Set("clientHash", local_hash) @@ -136,22 +126,27 @@ func (a *Requestor) write_handshake_request( buffer io.Writer ) (err error) { record.Set("meta", make(map[string]interface{})) codecHandshake, err := NewCodec(handshakeRequestshema) if err != nil { - return fmt.Errorf("Avro fail to get codec handshakeRequest %v",err) + err = fmt.Errorf("Avro fail to get codec handshakeRequest %v",err) + return } if a.send_protocol { json, err := a.local_protocol.Json() if err!=nil { - return err + return nil ,err } record.Set("clientProtocol", json) } + + if err = codecHandshake.Encode(buffer, record); err !=nil { - return fmt.Errorf("Encode handshakeRequest ",err) + err = fmt.Errorf("Encode handshakeRequest ",err) + return } - return nil + + return } func (a *Requestor) write_call_request(message_name string, request_datum interface{}, frame io.Writer) (err error) { diff --git a/requestor_test.go b/requestor_test.go index 98beac3..149c33b 100644 --- a/requestor_test.go +++ b/requestor_test.go @@ -6,18 +6,15 @@ import ( "bytes" "reflect" netty "github.com/sebglon/goavro/transceiver/netty" + "github.com/sebglon/goavro/transceiver" + "runtime" + "strconv" ) func TestWrite_handshake_request(t *testing.T) { //t.SkipNow() - rAddr, err := net.ResolveTCPAddr("tcp", "10.98.80.113:63001") - conn, err := net.DialTCP("tcp", nil, rAddr) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - transceiver, err := netty.NewTransceiver(netty.Config{}) + transceiver, err := netty.NewTransceiver(transceiver.Config{Host:HOST, Port:PORT}) if err != nil { t.Fatal(err) } @@ -27,11 +24,11 @@ func TestWrite_handshake_request(t *testing.T) { } requestor := NewRequestor(protocol, transceiver) - bb := new(bytes.Buffer) - requestor.write_handshake_request(bb) + + hds, err := requestor.write_handshake_request() // conn.Write(bb.Bytes()) - t.Logf("Handshake_request size %v %x\n",bb.Len(), bb.Bytes()) - t.Logf( "Handshake_request %v\n", bb.String()) + t.Logf("Handshake_request size %v %x\n",len(hds), hds) + t.Logf( "Handshake_request %v\n", hds) refHandshake := []byte("\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x00\x86\xaa\xda\xe2\xc4\x54\x74\xc0\xfe\x93\xff\xd0\xf2\x35\x0a\x65\x02\x00") //bytes := bb.Bytes() @@ -61,7 +58,7 @@ func TestRead_handshake_reponse(t *testing.T) { if err != nil { t.Fatal(err) } - record.Set("match", Enum{"match","BOTH"}) + record.Set("match", Enum{"match", "BOTH"}) record.Set("serverProtocol", nil) record.Set("serverHash", nil) record.Set("meta", nil) @@ -73,14 +70,12 @@ func TestRead_handshake_reponse(t *testing.T) { } t.Logf("Encode HandshakeResponse %v", bb.Bytes()) - _, err = codecHandshake.Decode(bytes.NewReader(bb.Bytes())) if err != nil { t.Fatal(err) } - - transceiver, err := netty.NewTransceiver(netty.Config{}) + transceiver, err := netty.NewTransceiver(transceiver.Config{Host:HOST, Port:PORT}) if err != nil { t.Fatal(err) } @@ -95,120 +90,150 @@ func TestRead_handshake_reponse(t *testing.T) { if err != nil { t.Fatal(err) } - -} - -type Conn struct { - bytes.Buffer -} -func (c *Conn) Close() error { - return nil } -func TestWrite_call_request(t *testing.T) { - //t.SkipNow() - +func TestWrite_call_request(t * testing.T) { + //t.SkipNow() + transceiver, err := netty.NewTransceiver(transceiver.Config{Host:HOST, Port:PORT}) + + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + bb := new(bytes.Buffer) + datum, err := protocol.NewRecord("AvroFlumeEvent") + if err != nil { + t.Fatal(err) + } + + headers := make(map[string]interface{}) + headers["host_header"] = "127.0.0.1" + datum.Set("headers", headers) + datum.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) + + requestor.write_call_request("append", datum, bb) + // conn.Write(bb.Bytes()) + t.Logf("\nCall_request size %v %v\n", bb.Len(), bb.Bytes()) + t.Logf("\nCall_request %v\n", bb.String()) + + codec, err := protocol.MessageRequestCodec("append") + if err != nil { + t.Fatal(err) + } + value, err := codec.Decode(bb) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(datum, value) { + t.Fatalf("Request not equals to ref %x, %x", datum, value) + } + } + +func TestWrite_call_requestHeader(t * testing.T) { + //t.SkipNow() + transceiver, err := netty.NewTransceiver(transceiver.Config{Host:HOST, Port:PORT}) + + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + bb := new(bytes.Buffer) + + requestor.write_call_requestHeader("append", bb) + + refHeader := []byte("\x00\x0c\x61\x70\x70\x65\x6e\x64") + bytes := bb.Bytes() + if !reflect.DeepEqual(refHeader, bytes) { + t.Fatalf("Request_Header not equals to ref %n%x, %n%x", len(refHeader), refHeader, len(bytes), bytes) + } + } + +func TestRead_call_responseMessage(t * testing.T) { + + transceiver, err := netty.NewTransceiver(transceiver.Config{Host:HOST, Port:PORT}) + + protocol, err := NewProtocol() + if err != nil { + t.Fatal(err) + } + requestor := NewRequestor(protocol, transceiver) + + codec, err := protocol.MessageResponseCodec("append") + if err != nil { + t.Fatal(err) + } + bb := new(bytes.Buffer) + codec.Encode(bb, Enum{"Status", "OK"}) + t.Logf("Bytes for OK %x", bb.Bytes()) + + err = requestor.read_call_responseMessage("append", bb) + if err != nil { + t.Fatal(err) + } + + codec.Encode(bb, Enum{"Status", "FAILED"}) + t.Logf("Bytes for FAILED %x", bb.Bytes()) + err = requestor.read_call_responseMessage("append", bb) + if err == nil || err.Error() != "Reponse failure. status == FAILED" { + t.Fatalf("Status FAILED can return error") + } + + } + + +const ( + RECV_BUF_LEN = 1024 + NETWORK = "tcp" + HOST = "127.0.0.1" + PORT=6666 + ADDR="127.0.0.1:6666" +) - transceiver, err := netty.NewTransceiver(netty.Config{}) - buf := &Conn{} - transceiver.Conn = buf - protocol, err := NewProtocol() - if err != nil { - t.Fatal(err) +func init() { + numProcs := runtime.NumCPU() + if numProcs < 2 { + numProcs = 2 } - requestor := NewRequestor(protocol, transceiver) + runtime.GOMAXPROCS(numProcs) - bb := new(bytes.Buffer) - datum, err := protocol.NewRecord("AvroFlumeEvent") + listener, err := net.Listen(NETWORK, "0.0.0.0:"+strconv.Itoa(PORT)) if err != nil { - t.Fatal(err) - } - - headers := make(map[string]interface{}) - headers["host_header"] = "127.0.0.1" - datum.Set("headers", headers) - datum.Set("body", []byte("2016-08-02 14:45:38|flume.composantTechnique_IS_UNDEFINED|flume.application_IS_UNDEFINED|flume.client_IS_UNDEFINED|flume.plateforme_IS_UNDEFINED|instance_IS_UNDEFINED|logname_IS_UNDEFINED|WARN |test.LogGenerator|test !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")) - - - - requestor.write_call_request("append", datum,bb) - // conn.Write(bb.Bytes()) - t.Logf("\nCall_request size %v %v\n",bb.Len(), bb.Bytes()) - t.Logf("\nCall_request %v\n", bb.String()) - - codec, err := protocol.MessageRequestCodec("append") - if err != nil { - t.Fatal(err) - } - value, err := codec.Decode(bb) - if err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(datum, value) { - t.Fatalf("Request not equals to ref %x, %x", datum, value) + println("error listening:", err.Error()) } + go func() { + for { + conn, err := listener.Accept() + if err != nil { + println("Error accept:", err.Error()) + return + } + go EchoFunc(conn) + } + }() } -func TestWrite_call_requestHeader(t *testing.T) { - //t.SkipNow() - transceiver, err := netty.NewTransceiver(netty.Config{}) - buf := &Conn{} - transceiver.Conn = buf - - protocol, err := NewProtocol() - if err != nil { - t.Fatal(err) - } - requestor := NewRequestor(protocol, transceiver) - - bb := new(bytes.Buffer) - - requestor.write_call_requestHeader("append", bb) - - refHeader := []byte("\x00\x0c\x61\x70\x70\x65\x6e\x64") - bytes := bb.Bytes() - if !reflect.DeepEqual(refHeader, bytes) { - t.Fatalf("Request_Header not equals to ref %n%x, %n%x", len(refHeader), refHeader, len(bytes), bytes) - } -} - -func TestRead_call_responseMessage(t *testing.T) { - - - transceiver, err := netty.NewTransceiver(netty.Config{}) - buf := &Conn{} - transceiver.Conn = buf - - protocol, err := NewProtocol() - if err != nil { - t.Fatal(err) +func EchoFunc(conn net.Conn) { + for { + buf := make([]byte, RECV_BUF_LEN) + n, err := conn.Read(buf) + if err != nil { + println("Error reading:", err.Error()) + return + } + println("received ", n, " bytes of data =", string(buf)) + n, err = conn.Write(buf) + if err != nil { + println("Error writing:", err.Error()) + return + } + println("sended ", n, " bytes of data =", string(buf)) } - requestor := NewRequestor(protocol, transceiver) - - - codec, err := protocol.MessageResponseCodec("append") - if err != nil { - t.Fatal(err) - } - bb := new(bytes.Buffer) - codec.Encode(bb, Enum{"Status", "OK"}) - t.Logf("Bytes for OK %x",bb.Bytes() ) - - - err = requestor.read_call_responseMessage("append", bb) - if err != nil { - t.Fatal(err) - } - - codec.Encode(bb, Enum{"Status", "FAILED"}) - t.Logf("Bytes for FAILED %x",bb.Bytes() ) - err = requestor.read_call_responseMessage("append", bb) - if err == nil || err.Error() != "Reponse failure. status == FAILED"{ - t.Fatalf("Status FAILED can return error") - } - } diff --git a/transceiver/Tranceiver.go b/transceiver/Tranceiver.go index bf2dffb..3a36ed1 100644 --- a/transceiver/Tranceiver.go +++ b/transceiver/Tranceiver.go @@ -3,9 +3,32 @@ package transceiver import ( "bytes" "io" + "time" ) +type WriteHandshake func() ([]byte, error) +type ReadHandshake func(io.Reader) (bool, error) type Transceiver interface { Transceive(request []bytes.Buffer) ([]io.Reader, error) + InitHandshake(WriteHandshake, ReadHandshake ) + + + +} + + + +type Config struct { + Port int `json:"port"` + Host string `json:"host"` + Network string `json:"network"` + SocketPath string `json:"socket_path"` + Timeout time.Duration `json:"timeout"` + AsyncConnect bool `json:"async_connect"` + BufferLimit int `json:"buffer_limit"` + RetryWait int `json:"retry_wait"` + MaxRetry int `json:"max_retry"` + InitialCap int `json:"initial_cap"` + MaxCap int `json:"max_cap"` +} -} \ No newline at end of file diff --git a/transceiver/connection.go b/transceiver/connection.go new file mode 100644 index 0000000..886a2c2 --- /dev/null +++ b/transceiver/connection.go @@ -0,0 +1,46 @@ +package transceiver + +import ( + "io" + "net" + "strconv" +) + +type HandShakeConn interface { + io.ReadWriteCloser + IsChecked() bool + Checked(bool) + GetConn() (net.Conn, error) +} + +type Connection struct { + net.Conn + checked bool + bad bool +} + + +func NewConnection(config Config) (*Connection, error) { + + conn := &Connection{} + var err error + switch config.Network { + case "tcp": + conn.Conn, err = net.DialTimeout(config.Network, config.Host+":"+strconv.Itoa(config.Port), config.Timeout) + case "unix": + conn.Conn, err = net.DialTimeout(config.Network, config.SocketPath, config.Timeout) + default: + err = net.UnknownNetworkError(config.Network) + } + + return conn, err +} + +func (c *Connection) Checked(check bool) { + c.checked = check +} + +func (c *Connection) IsChecked() bool{ + return c.checked +} + diff --git a/transceiver/netty/Netty.go b/transceiver/netty/Netty.go index 9b66ac4..d400d60 100644 --- a/transceiver/netty/Netty.go +++ b/transceiver/netty/Netty.go @@ -2,108 +2,85 @@ package netty import ( "bytes" - "net" "encoding/binary" - "fmt" "io" "sync" - "time" - "strconv" - "math" - "os" + "github.com/sebglon/goavro/transceiver" + "fmt" ) +type NettyTransceiver struct { + transceiver.Pool + pool *transceiver.Pool + mu sync.Mutex + pending []byte + alreadyCalled bool + writeHandShake transceiver.WriteHandshake + readHandshake transceiver.ReadHandshake +} +func NewTransceiver(config transceiver.Config) (f* NettyTransceiver, err error){ + f = &NettyTransceiver{} + f.pool, err = transceiver.NewPool(config) + return +} +func (t NettyTransceiver) InitHandshake(writer transceiver.WriteHandshake,reader transceiver.ReadHandshake ) { + t.writeHandShake=writer + t.readHandshake=reader +} -const ( - defaultHost = "127.0.0.1" - defaultNetwork = "tcp" - defaultSocketPath = "" - defaultPort = 63001 - defaultTimeout = 3 * time.Second - defaultBufferLimit = 8 * 1024 * 1024 - defaultRetryWait = 500 - defaultMaxRetry = 13 - defaultReconnectWaitIncreRate = 1.5 -) -type Config struct { - NettyPort int `json:"netty_port"` - NettyHost string `json:"netty_host"` - NettyNetwork string `json:"netty_network"` - NettySocketPath string `json:"netty_socket_path"` - Timeout time.Duration `json:"timeout"` - AsyncConnect bool `json:"async_connect"` - BufferLimit int `json:"buffer_limit"` - RetryWait int `json:"retry_wait"` - MaxRetry int `json:"max_retry"` -} +func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ + nettyFrame := new(bytes.Buffer) + t.Pack(nettyFrame, requests) + conn, pc, err := t.Pool.Conn() -type NettyTransceiver struct { - Config - Conn io.ReadWriteCloser - reconnecting bool - mu sync.Mutex - pending []byte -} -func NewTransceiver(config Config) (f* NettyTransceiver, err error){ - if config.NettyNetwork == "" { - config.NettyNetwork = defaultNetwork - } - if config.NettyHost == "" { - config.NettyHost = defaultHost - } - if config.NettyPort == 0 { - config.NettyPort = defaultPort - } - if config.NettySocketPath == "" { - config.NettySocketPath = defaultSocketPath - } - if config.Timeout == 0 { - config.Timeout = defaultTimeout + if err!=nil { + return nil, err } - if config.BufferLimit == 0 { - config.BufferLimit = defaultBufferLimit - } - if config.RetryWait == 0 { - config.RetryWait = defaultRetryWait - } - if config.MaxRetry == 0 { - config.MaxRetry = defaultMaxRetry + defer pc.Close() + + if !conn.IsChecked() { + frame0 := requests[0] + handshake, err := t.writeHandShake() + if err!=nil { + return nil, err + } + + requests[0].Reset() + _, err = requests[0].Write(append( handshake, frame0.Bytes()...)) + if err!=nil { + return nil, err + } } - if config.AsyncConnect { - f = &NettyTransceiver{Config: config, reconnecting: true} - f.reconnect() - } else { - f = &NettyTransceiver{Config: config, reconnecting: false} - err = f.connect() + + bodyBytes, err := t.pool.Call(conn, pc, nettyFrame.Bytes()) + if err != nil { + return nil, err } - return -} -func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ - nettyFrame := new(bytes.Buffer) - t.Pack(nettyFrame, requests) - // Send request - t.pending = append(t.pending, nettyFrame.Bytes()...) - if err := t.send(); err != nil { - t.close() - if len(t.pending) > t.Config.BufferLimit { - t.flushBuffer() + + resps, err := t.Unpack(bodyBytes) + if err != nil { + return nil, err + } + + if !conn.IsChecked() { + ok, err := t.readHandshake(resps[0]) + if err!=nil { + return nil, err + } + conn.Checked(ok) + if !ok { + return nil, fmt.Errorf("Fail to validate Handshake") } + return resps[1:], nil } else { - t.flushBuffer() + return resps, nil } - // Read Response - bodyBytes := make([]byte, 1024) - t.receive(bodyBytes) -// if err!=nil { -// return nil, fmt.Errorf("Fail to read on socket %v", err) -// } - return t.Unpack(bodyBytes) } func (t *NettyTransceiver) Pack(frame *bytes.Buffer, requests []bytes.Buffer) { @@ -142,110 +119,3 @@ func (t *NettyTransceiver) Unpack(frame []byte) ([]io.Reader, error) { return result, nil } - -// connect establishes a new connection using the specified transport. -func (f *NettyTransceiver) connect() (error) { - var err error - switch f.Config.NettyNetwork { - case "tcp": - f.Conn, err = net.DialTimeout(f.Config.NettyNetwork, f.Config.NettyHost+":"+strconv.Itoa(f.Config.NettyPort), f.Config.Timeout) - case "unix": - f.Conn, err = net.DialTimeout(f.Config.NettyNetwork, f.Config.NettySocketPath, f.Config.Timeout) - default: - err = net.UnknownNetworkError(f.Config.NettyNetwork) - } - return err -} - - -func e(x, y float64) int { - return int(math.Pow(x, y)) -} - -func (f *NettyTransceiver) reconnect() { - go func() { - - for i := 0; ; i++ { - - err := f.connect() - if err == nil { - f.mu.Lock() - f.reconnecting = false - f.mu.Unlock() - break - } else { - if i == f.Config.MaxRetry { - panic("Netty#reconnect: failed to reconnect!") - } - waitTime := f.Config.RetryWait * e(defaultReconnectWaitIncreRate, float64(i-1)) - time.Sleep(time.Duration(waitTime) * time.Millisecond) - } - } - }() -} - -func (f *NettyTransceiver) flushBuffer() { - f.mu.Lock() - defer f.mu.Unlock() - f.pending = f.pending[0:0] -} - - -// Close closes the connection. -func (f *NettyTransceiver) Close() (err error) { - if len(f.pending) > 0 { - err = f.send() - } - f.close() - return -} - -// close closes the connection. -func (f *NettyTransceiver) close() (err error) { - if f.Conn != nil { - f.mu.Lock() - defer f.mu.Unlock() - } else { - return - } - if f.Conn != nil { - f.Conn.Close() - f.Conn = nil - } - return -} - -func (f *NettyTransceiver) send() (err error) { - if f.Conn == nil { - if f.reconnecting == false { - f.mu.Lock() - f.reconnecting = true - f.mu.Unlock() - f.reconnect() - } - err = fmt.Errorf("Netty#send: can't send logs, client is reconnecting") - } else { - f.mu.Lock() - _, err = f.Conn.Write(f.pending) - f.mu.Unlock() - } - return -} - -func (f *NettyTransceiver) receive(resp []byte) (err error) { - - if f.Conn == nil { - if f.reconnecting == false { - f.mu.Lock() - f.reconnecting = true - f.mu.Unlock() - f.reconnect() - } - err = fmt.Errorf("Netty#receive: can't send logs, client is reconnecting") - } else { - f.mu.Lock() - _, err = f.Conn.Read(resp) - f.mu.Unlock() - } - return -} \ No newline at end of file diff --git a/transceiver/netty/Netty_test.go b/transceiver/netty/Netty_test.go index 7b8de8b..6e6de40 100644 --- a/transceiver/netty/Netty_test.go +++ b/transceiver/netty/Netty_test.go @@ -8,19 +8,19 @@ import ( "io/ioutil" "runtime" "net" + "github.com/sebglon/goavro/transceiver" + + "strconv" ) const ( RECV_BUF_LEN = 1024 + NETWORK = "tcp" + HOST = "127.0.0.1" + PORT=6666 + ADDR="127.0.0.1:6666" ) -type Conn struct { - bytes.Buffer -} - -func (c *Conn) Close() error { - return nil -} func init() { numProcs := runtime.NumCPU() @@ -29,7 +29,7 @@ func init() { } runtime.GOMAXPROCS(numProcs) - listener, err := net.Listen("tcp", "0.0.0.0:6666") + listener, err := net.Listen(NETWORK, "0.0.0.0:"+strconv.Itoa(PORT)) if err != nil { println("error listening:", err.Error()) } @@ -54,14 +54,21 @@ func EchoFunc(conn net.Conn) { return } println("received ", n, " bytes of data =", string(buf)) + n, err = conn.Write(buf) + if err != nil { + println("Error writing:", err.Error()) + return + } + println("sended ", n, " bytes of data =", string(buf)) } } func TestTransceive(t *testing.T) { - f := &NettyTransceiver{Config: Config{}, reconnecting: false} + f, err := NewTransceiver(transceiver.Config{Network:NETWORK, SocketPath:ADDR, Host:HOST, Port:PORT}) + if err != nil { + t.Fatal(err) + } - buf := &Conn{} - f.Conn = buf msg := "This is test writing." bmsg := make([]bytes.Buffer, 1) @@ -69,9 +76,8 @@ func TestTransceive(t *testing.T) { resp, err := f.Transceive(bmsg) if err != nil { - t.Error(err) + t.Fatal(err.Error()) } - brcv := make([]byte, len([]byte(msg))) resp[0].Read(brcv) rcv := string(brcv) diff --git a/transceiver/pool.go b/transceiver/pool.go new file mode 100644 index 0000000..5d05edf --- /dev/null +++ b/transceiver/pool.go @@ -0,0 +1,135 @@ +package transceiver + +import ( + "gopkg.in/fatih/pool.v2" + "net" + "fmt" + "sync" + "errors" + "log" + "time" +) + + +var ( + errPoolClosed = errors.New("Avro transceiver: Pool Closed") +) +type Pool struct { + Config + pool pool.Pool + mu sync.RWMutex + closed bool +} + +const ( + defaultHost = "127.0.0.1" + defaultNetwork = "tcp" + defaultSocketPath = "" + defaultPort = 63001 + defaultTimeout = 3 * time.Second + defaultBufferLimit = 8 * 1024 * 1024 + defaultRetryWait = 500 + defaultMaxRetry = 13 + defaultInitialCap = 2 + defaultMaxCap = 5 + defaultReconnectWaitIncreRate = 1.5 +) + +func NewPool(config Config) (*Pool, error) { + if config.Network == "" { + config.Network = defaultNetwork + } + if config.Host == "" { + config.Host = defaultHost + } + if config.Port == 0 { + config.Port = defaultPort + } + if config.SocketPath == "" { + config.SocketPath = defaultSocketPath + } + if config.Timeout == 0 { + config.Timeout = defaultTimeout + } + if config.BufferLimit == 0 { + config.BufferLimit = defaultBufferLimit + } + if config.RetryWait == 0 { + config.RetryWait = defaultRetryWait + } + if config.MaxRetry == 0 { + config.MaxRetry = defaultMaxRetry + } + if config.InitialCap == 0 { + config.InitialCap = defaultInitialCap + } + if config.MaxCap == 0 { + config.MaxCap = defaultMaxCap + } + p, err := pool.NewChannelPool(config.InitialCap,config.MaxCap, func() (net.Conn, error) { + conn, err := NewConnection(config) + if err != nil { + return nil, fmt.Errorf("\nFail to init connec, %#v \n%v",config,err) + } + return conn, err + }) + if err != nil { + return nil, err + } + return &Pool{ + pool: p, + Config: config, + }, nil + +} + +func (p *Pool) Conn() (*Connection, *pool.PoolConn, error) { + p.mu.RLock() + defer p.mu.RUnlock() + + if p.closed { + return nil, nil, errPoolClosed + } + + + nc, err := p.pool.Get() + if err != nil { + return nil, nil, err + } + + log.Printf(" %T %#v", nc,nc) + + pc, ok := nc.(*pool.PoolConn) + if !ok { + // This should never happen! + return nil, nil, fmt.Errorf("Invalid connection in pool") + } + + conn, ok := pc.Conn.(*Connection) + if !ok { + // This should never happen! + return nil, nil, fmt.Errorf("Invalid connection in pool") + } + + return conn, pc, nil +} + +func (p *Pool) Call(conn *Connection, pc *pool.PoolConn, req []byte) (resp []byte, err error) { + if err != nil { + return + } + defer pc.Close() + + _,err= conn.Write(req) + + if err != nil { + return nil, err + } + resp = make([]byte, 1024) + _,err = conn.Read(resp) + + if err != nil { + return nil, err + } + return +} \ No newline at end of file From 279575f00737ac416eba978a823b9af3cc310070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Tue, 6 Sep 2016 16:58:20 +0200 Subject: [PATCH 14/15] Correct test failed --- transceiver/netty/Netty.go | 20 ++++++++++++++------ transceiver/netty/Netty_test.go | 8 +++++--- transceiver/pool.go | 7 +++++-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/transceiver/netty/Netty.go b/transceiver/netty/Netty.go index d400d60..a65c6e5 100644 --- a/transceiver/netty/Netty.go +++ b/transceiver/netty/Netty.go @@ -7,11 +7,11 @@ import ( "sync" "github.com/sebglon/goavro/transceiver" "fmt" + "log" ) type NettyTransceiver struct { transceiver.Pool - pool *transceiver.Pool mu sync.Mutex pending []byte alreadyCalled bool @@ -20,20 +20,25 @@ type NettyTransceiver struct { } func NewTransceiver(config transceiver.Config) (f* NettyTransceiver, err error){ f = &NettyTransceiver{} - f.pool, err = transceiver.NewPool(config) + pool , err := transceiver.NewPool(config) + if err !=nil { + return + } + f.Pool =*pool return } -func (t NettyTransceiver) InitHandshake(writer transceiver.WriteHandshake,reader transceiver.ReadHandshake ) { +func (t *NettyTransceiver) InitHandshake(writer transceiver.WriteHandshake,reader transceiver.ReadHandshake ) { t.writeHandShake=writer t.readHandshake=reader } -func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ +func (t *NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ nettyFrame := new(bytes.Buffer) t.Pack(nettyFrame, requests) + log.Printf("%#v",t.Pool) conn, pc, err := t.Pool.Conn() if err!=nil { @@ -43,6 +48,9 @@ func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, erro if !conn.IsChecked() { frame0 := requests[0] + if t.writeHandShake ==nil { + return nil, fmt.Errorf("InitHandshake not called before Transceive") + } handshake, err := t.writeHandShake() if err!=nil { return nil, err @@ -55,7 +63,7 @@ func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, erro } } - bodyBytes, err := t.pool.Call(conn, pc, nettyFrame.Bytes()) + bodyBytes, err := t.Pool.Call(conn, pc, nettyFrame.Bytes()) if err != nil { return nil, err } @@ -67,7 +75,7 @@ func (t NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, erro return nil, err } - if !conn.IsChecked() { + if !conn.IsChecked() && len(resps)>1{ ok, err := t.readHandshake(resps[0]) if err!=nil { return nil, err diff --git a/transceiver/netty/Netty_test.go b/transceiver/netty/Netty_test.go index 6e6de40..123283b 100644 --- a/transceiver/netty/Netty_test.go +++ b/transceiver/netty/Netty_test.go @@ -9,8 +9,8 @@ import ( "runtime" "net" "github.com/sebglon/goavro/transceiver" - "strconv" + "io" ) const ( @@ -64,10 +64,11 @@ func EchoFunc(conn net.Conn) { } func TestTransceive(t *testing.T) { - f, err := NewTransceiver(transceiver.Config{Network:NETWORK, SocketPath:ADDR, Host:HOST, Port:PORT}) + f, err := NewTransceiver(transceiver.Config{Network:NETWORK, Host:HOST, Port:PORT}) if err != nil { t.Fatal(err) } + f.InitHandshake(func()([]byte, error){return make([]byte,1), nil},func(io.Reader)(bool, error){return true, nil}) msg := "This is test writing." @@ -126,4 +127,5 @@ func TestUnpack(t *testing.T) { t.Fatalf("Reponse message not equals (0) %x/%x; (1) %x/%x","\x0a", respons[0], "\x0b", respons[1]) } -} \ No newline at end of file +} + diff --git a/transceiver/pool.go b/transceiver/pool.go index 5d05edf..0951f9e 100644 --- a/transceiver/pool.go +++ b/transceiver/pool.go @@ -76,10 +76,13 @@ func NewPool(config Config) (*Pool, error) { if err != nil { return nil, err } - return &Pool{ + + pool := &Pool{ pool: p, Config: config, - }, nil + } + log.Printf("%#v",pool.pool) + return pool, nil } From c32db8962dc1c044957b63f798a01ecf72c02c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20GLON?= Date: Tue, 6 Sep 2016 17:12:02 +0200 Subject: [PATCH 15/15] add close pool funtion --- transceiver/Tranceiver.go | 1 + transceiver/netty/Netty.go | 2 ++ transceiver/netty/Netty_test.go | 1 + transceiver/pool.go | 3 +++ 4 files changed, 7 insertions(+) diff --git a/transceiver/Tranceiver.go b/transceiver/Tranceiver.go index 3a36ed1..37767e0 100644 --- a/transceiver/Tranceiver.go +++ b/transceiver/Tranceiver.go @@ -11,6 +11,7 @@ type ReadHandshake func(io.Reader) (bool, error) type Transceiver interface { Transceive(request []bytes.Buffer) ([]io.Reader, error) InitHandshake(WriteHandshake, ReadHandshake ) + Close() diff --git a/transceiver/netty/Netty.go b/transceiver/netty/Netty.go index a65c6e5..663a887 100644 --- a/transceiver/netty/Netty.go +++ b/transceiver/netty/Netty.go @@ -35,6 +35,8 @@ func (t *NettyTransceiver) InitHandshake(writer transceiver.WriteHandshake,reade + + func (t *NettyTransceiver) Transceive(requests []bytes.Buffer) ([]io.Reader, error){ nettyFrame := new(bytes.Buffer) t.Pack(nettyFrame, requests) diff --git a/transceiver/netty/Netty_test.go b/transceiver/netty/Netty_test.go index 123283b..2099595 100644 --- a/transceiver/netty/Netty_test.go +++ b/transceiver/netty/Netty_test.go @@ -68,6 +68,7 @@ func TestTransceive(t *testing.T) { if err != nil { t.Fatal(err) } + defer f.Close() f.InitHandshake(func()([]byte, error){return make([]byte,1), nil},func(io.Reader)(bool, error){return true, nil}) diff --git a/transceiver/pool.go b/transceiver/pool.go index 0951f9e..0270955 100644 --- a/transceiver/pool.go +++ b/transceiver/pool.go @@ -135,4 +135,7 @@ func (p *Pool) Call(conn *Connection, pc *pool.PoolConn, req []byte) (resp []byt return nil, err } return +} +func (t *Pool) Close() { + t.pool.Close() } \ No newline at end of file