178 lines
6.8 KiB
Markdown
Executable File
178 lines
6.8 KiB
Markdown
Executable File
author: breel@qualtrics.com
|
|
|
|
Qualtrics Messaging Platform Go Client
|
|
==================================
|
|
|
|
### Installation
|
|
|
|
`go get -u gitlab.app-eng.qops.net/golang/qmp/...`
|
|
|
|
---
|
|
|
|
### Limitations
|
|
|
|
The Golang QMP library is pure Go (hooray!). Unfortunately, avoiding Kafka's native libraries comes with some limitations.
|
|
|
|
#### Vendoring
|
|
|
|
* The `github.com/satori/go.uuid` in the library assumes no later than `v1.1.0`.
|
|
|
|
#### QSL
|
|
|
|
* The Go QSL library does not support fast filtering, partial decoding, or aliases. It does support full decoding and re-encoding into a smaller, compatible schema.
|
|
* The Go QSL library produces asynchronously. QSL Messages are not safe to modify until a Producer reaches its callback.
|
|
* The schema spec compatibility checking does not play nicely with complex schema declarations. It cannot interpret the compatibility of a union with any schema spec.
|
|
* The Qualtrics Schema Registry (maintained by the core QMP team) injects namespaces into its schemas. If your schema isn't processing with the Go library, try fetching the registry version and looking for injected namespaces. Check by visiting a variety of `http://qmp-schema-registry.service.consul:8081/subjects/qmp.configuration-value/versions/1`.
|
|
* You must explicitly list the type of any Avro union. For example, if your schema has `{"type": ["null", "custom"]}`, then you must set its value with `qsl.Union("namespace.custom", value)`.
|
|
|
|
#### QPCL
|
|
|
|
* You must .Close() all Producers and .Stop() all Consumers or they will leak memory and you're gonna have a bad time.
|
|
* All callback executions are *synchronous* to a Consumer or Producer's message processing. Only one message is processed at a time. If your callback takes time, consider creating a goroutine in the callback.
|
|
* Editing a QMessage before a Producer reaches its callback will change the callback. It will not change the published message.
|
|
|
|
---
|
|
|
|
### Client Usage
|
|
|
|
#### Initialize
|
|
|
|
The client library first needs to be 'registered'. This just tells qmp what
|
|
version of the library you are using which helps track usage and manage
|
|
deprecation.
|
|
|
|
The provided client-name should just be the name of your service.
|
|
|
|
```golang
|
|
import qpcl "gitlab-app.eng.qops.net/golang/qmp"
|
|
import qsl "gitlab-app.eng.qops.net/golang/qmp/qsl"
|
|
|
|
// Register the serialization library
|
|
err := qsl.RegisterApplication("my-client-name")
|
|
if err != nil { /* handle */ }
|
|
|
|
// Register the producer/consumer library
|
|
err := qpcl.RegisterApplication("my-client-name")
|
|
if err != nil { /* handle */ }
|
|
```
|
|
|
|
#### Consume Messages
|
|
|
|
```golang
|
|
qc, err := qpcl.NewConsumer(qpcl.CBulk, "topic-name", "group-id", qReader, numberThreads)
|
|
if err != nil { /* handle */ }
|
|
|
|
qc.Start(func(qMessage qsl.Message){
|
|
/* handler for received messages */
|
|
fmt.Println("Message received with Trace ID", qMessage.GetTraceID())
|
|
})
|
|
|
|
// ...
|
|
|
|
err = qc.Stop()
|
|
if err != nil { /* handle */ }
|
|
```
|
|
|
|
#### Produce Messages
|
|
|
|
```golang
|
|
qp, err := qpcl.NewProducer(qpcl.PBulk, "topic-name")
|
|
if err != nil { /* handle */ }
|
|
|
|
qp.Put(qMessage, func(err error, qMessage qsl.Message) {
|
|
/* handler for sent messages */
|
|
fmt.Println("Message sent with Trace ID", qMessage.GetTraceID(), "and error", err)
|
|
})
|
|
|
|
// ...
|
|
|
|
err = qp.Close()
|
|
if err != nil { /* handle */ }
|
|
```
|
|
|
|
---
|
|
|
|
### Message Usage
|
|
|
|
#### Initialize
|
|
|
|
```golang
|
|
import qsl "gitlab-app.eng.qops.net/golang/qmp/qsl"
|
|
|
|
// ...
|
|
|
|
err := qsl.RegisterApplication("my-client-name")
|
|
if err != nil { /* handle */ }
|
|
```
|
|
|
|
#### Read Messages
|
|
|
|
```golang
|
|
qr, err := qsl.NewReader("schema.name.v1")
|
|
if err != nil { /* handle */ }
|
|
|
|
// ...
|
|
|
|
qm, err := qr.NewMessage(encodedAvroObject)
|
|
if err != nil { /* handle */ }
|
|
```
|
|
|
|
#### Write Messages
|
|
|
|
```golang
|
|
qw, err := qsl.NewWriter("schema.name.v1")
|
|
if err != nil { /* handle */ }
|
|
|
|
// Blank QMessage from writer's schema
|
|
msg, err := qw.NewMessage()
|
|
if err != nil { /* handle */ }
|
|
msg.SetTraceID("Look at me, I'm Mr. Meeseeks")
|
|
|
|
// QMessage from a QMessage compatible with writer's schema
|
|
msg, err = qw.NewFromQMessage(msg)
|
|
if err != nil { /* handle */ }
|
|
|
|
// QMessage from a map of the writer schema's fields and their values
|
|
msg, err = qw.NewFromObject(map[string]interface{}{"schema_id":"my.schema.v1"})
|
|
if err != nil { /* handle */ }
|
|
```
|
|
|
|
#### Message Quick Reference
|
|
|
|
[Find the full specification for QMessages here.](http://godoc-app.eng.qops.net/gitlab-app.eng.qops.net/golang/qmp/qsl)
|
|
|
|
```golang
|
|
mapped := msg.GetAvroObject() // map[string]interface{} representation of the message
|
|
bytes, err := msg.GetSerializedBytes() // []byte raw avro value
|
|
writerSchemaSpec := msg.GetWriterSchema() // string writer schema from schema registry
|
|
messageID, err := msg.GetMessageID() // string uuid for the message
|
|
timestamp, err := msg.GetTimestamp() // int64 timestamp of message creation
|
|
writerHost, err := msg.GetWriterHost() // string often unhelpful
|
|
writerApp, err := msg.GetWriterApp() // string of the client's string passed to RegisterApplication()
|
|
traceID, err := msg.GetTraceID() // string uuid of a series of transactions
|
|
orgID, err := msg.GetOrganizationID() // string of qualtrics or a team
|
|
|
|
err := SetWriterSchema("schema.id.v1", "{type: something}") // change the message's serialization schema
|
|
err := SetOrganizationID("hello from sp1") // change the message's org ID
|
|
err := SetWriterHost("az1") // change the message's writer host
|
|
err := SetTraceID("follow me down the rabbit hole") // replace the transaction ID with another string
|
|
|
|
value, err := msg.Get("key") // interface{} of the payload value named "key"
|
|
err := msg.Set("key", qsl.Union("namespace.name", value)) // set the value of the payload at "key"
|
|
|
|
mapped, err := msg.GetPayload() // map[string]interface{} of the payload
|
|
err := msg.SetPayload(mapped) // set the payload yourself. Better match your schema!
|
|
|
|
bytes, err := msg.GetOriginalBytes() // []byte if the message was consumed/read in
|
|
readerSchemaSpec, err := msg.GetReaderSchema() // string of the schema if the message was consumed/read in
|
|
err := UseOriginalBytes(true) // Produce the raw avro value if the message was consumed/read in. Use to forward a QMessage.
|
|
yesOrNo := msg.UsingOriginalBytes() // bool whether the message is forwarding or not if the message was consumed/read in
|
|
```
|
|
|
|
---
|
|
|
|
### Resources
|
|
|
|
* [GoDoc](http://godoc-app.eng.qops.net/gitlab-app.eng.qops.net/golang/qmp)
|
|
* [QSL Specification](https://docs.google.com/document/d/1Xmlo9Wj7i1fA24YO4UzQXet0Yk4OldpEJ1TJvS5D7sU/edit)
|