debug = tornado.options.options.environment == 'dev' if debug: logging.getLogger().setLevel(logging.DEBUG) app_settings = { 'debug': debug, 'login_url': 'dummy', 'apns_conn': lib.connection.APNSConn() } /<service>/<service>_api.py, 2 of 4 set up logging and app settings, then we (define our)
class BaseHandler(tornado.web.RequestHandler): def error(self, status_code, status_txt, data=None): """write an api error in the appropriate response format""" self.api_response(status_code=status_code, status_txt=status_txt, data=data) def api_response(self, data, status_code=200, status_txt="OK"): """write an api response in json""" self.set_header("Content-‐Type", "application/json; charset=utf-‐8") self.finish(json.dumps(dict(data=data, status_code=status_code, status_txt=status_txt))) /<service>/app/basic.py writing our responses in our standard API response format, (including)
class BaseHandler(tornado.web.RequestHandler): def error(self, status_code, status_txt, data=None): """write an api error in the appropriate response format""" self.api_response(status_code=status_code, status_txt=status_txt, data=data) def api_response(self, data, status_code=200, status_txt="OK"): """write an api response in json""" self.set_header("Content-‐Type", "application/json; charset=utf-‐8") self.finish(json.dumps(dict(data=data, status_code=status_code, status_txt=status_txt))) /<service>/app/basic.py errors, and our (successful API responses as well)
class BaseHandler(tornado.web.RequestHandler): def error(self, status_code, status_txt, data=None): """write an api error in the appropriate response format""" self.api_response(status_code=status_code, status_txt=status_txt, data=data) def api_response(self, data, status_code=200, status_txt="OK"): """write an api response in json""" self.set_header("Content-‐Type", "application/json; charset=utf-‐8") self.finish(json.dumps(dict(data=data, status_code=status_code, status_txt=status_txt))) /<service>/app/basic.py
class BaseHandler(tornado.web.RequestHandler): def error(self, status_code, status_txt, data=None): """write an api error in the appropriate response format""" self.api_response(status_code=status_code, status_txt=status_txt, data=data) def api_response(self, data, status_code=200, status_txt="OK"): """write an api response in json""" self.set_header("Content-‐Type", "application/json; charset=utf-‐8") self.finish(json.dumps(dict(data=data, status_code=status_code, status_txt=status_txt))) /<service>/app/basic.py setting content type to JSON. In (api.py)
@tornado.web.asynchronous def get(self): unit = self.get_unit_argument() total_key = self.get_argument("key") self.get_all_units(total_key, user_callback=self.async_callback(self.finish_list, unit=unit)) /<service>/app/api.py and sets up a (callback)
@tornado.web.asynchronous def get(self): unit = self.get_unit_argument() total_key = self.get_argument("key") self.get_all_units(total_key, user_callback=self.async_callback(self.finish_list, unit=unit)) /<service>/app/api.py self.finish_list(). Older Tornado code, similar to Twisted or default node.js last time I looked.
data, unit): if data is None: raise tornado.web.HTTPError(500, 'UNKNOWN_ERROR') units = self.get_int_argument("units", 1) unit_reference_dt = self.get_unit_reference_datetime() output = ListHandler.get_output_counts(data, unit, units, unit_reference_dt) logging.debug("output: %r" % output) self.api_response(output) /<service>/app/api.py accepts the response from the callback and if successful, (outputs)
data, unit): if data is None: raise tornado.web.HTTPError(500, 'UNKNOWN_ERROR') units = self.get_int_argument("units", 1) unit_reference_dt = self.get_unit_reference_datetime() output = ListHandler.get_output_counts(data, unit, units, unit_reference_dt) logging.debug("output: %r" % output) self.api_response(output) /<service>/app/api.py response in standard API response format. A (modern version)
@gen.coroutine def get(self): unit = self.get_unit_argument() total_key = self.get_argument("key") data = yield self.get_all_units(total_key) units = self.get_int_argument("units", 1) unit_reference_dt = self.get_unit_reference_datetime() output = ListHandler.get_output_counts(data, unit, units, unit_reference_dt) logging.debug("output: %r" % output) /<service>/app/api.py, Modernized would use the gen.coroutine decorator and yield on the async call. The (lib directory)
hold whatever utility scripts make sense, including nagios check scripts. Conventions help build a service that is easily maintained, particularly at 3am. Operational simplification. Implementing a Bitly Service (in Go)
eshistoryClient *eshistory.Client blacklist = NewGreyList("../conf/blacklist.dat") whitelist = NewGreyList("../conf/whitelist.dat") ) eshistory/eshistory_api/eshistory_api.go Global variables, including reading in our blacklist data
environment = flag.String("environment", "dev", "run environment type") settingsFile = flag.String("settings-‐file", "../settings.json", "path to settings.json file") port = flag.Int("port", 7850, "port to listen on for http requests") verbose = flag.Bool("verbose", false, "verbose logging") logRequests = flag.Bool("log-‐requests", false, "turn on logging requests in Common Log Format") logMemoryStats = flag.Bool("log-‐memory-‐stats", false, "turn on logging of memory statistics") ) flag.Parse() settings.Load(*settingsFile, *environment) roles.LoadRoles("/bitly/local/conf/roles.json") //gpg = settings.OpenGpgSettingsFromSettingsFile(*settingsFile) if *verbose { log.SetLevel(log.DebugLevel) log.Info("Verbose logging enabled") *logRequests = true *logMemoryStats = true } else { switch settings.GetString("log_level") { case "debug": eshistory/eshistory_api/eshistory_api.go logRequests flag we’ll see in action later, but adds standard request logging (~common log format) to our application — that does *not* come in Go standard library.
log.SetLevel(log.DebugLevel) case "info": log.SetLevel(log.InfoLevel) case "error": log.SetLevel(log.ErrorLevel) default: log.SetLevel(log.WarnLevel) } } eshistoryClient = &eshistory.Client{ Conn: eshistory.InitElasticsearchConnection("eshistory_elasticsearch"), IndexName: settings.GetString("es_index_name"), IndexType: settings.GetString("es_index_type"), MC: memcache.NewClient(settings.GetRole("memcached").Entries()), StatsdQueue: statsd.InitStatsd(), } eshistory/eshistory_api/eshistory_api.go The remants of setting the log level and (initializing the connection to our datastore)
listener, err := net.Listen("tcp", listenAddr) if err != nil { log.Fatalf("FATAL: Listen on %s has failed: %s", listenAddr, err) } exitChan := exitchan.New() go func() { <-‐exitChan err := listener.Close() if err != nil { log.Fatalf("FATAL: got signal but failed to close listener: %s", err) } }() log.Infof("logMemoryStats: %s", *logMemoryStats) if *logMemoryStats { eshistory/eshistory_api/eshistory_api.go setting up our connection to statsd, establishing our TCP listener on the (provided port)
listener, err := net.Listen("tcp", listenAddr) if err != nil { log.Fatalf("FATAL: Listen on %s has failed: %s", listenAddr, err) } exitChan := exitchan.New() go func() { <-‐exitChan err := listener.Close() if err != nil { log.Fatalf("FATAL: got signal but failed to close listener: %s", err) } }() log.Infof("logMemoryStats: %s", *logMemoryStats) if *logMemoryStats { eshistory/eshistory_api/eshistory_api.go and setting up the response to a signal. Next, (we create a net/http Handler for routing)
go memstats.MemStats() } var router http.Handler log.Infof("logRequests: %s", *logRequests) if *logRequests { router = request_logger.LoggingHandler(os.Stdout, routes.NewRouter(app_routes)) } else { router = routes.NewRouter(app_routes) } httpServer := &http.Server{ Handler: router, // MaxHeaderBytes: MaxReqHeaderBytes, } log.Infof("INFO: Listening for requests on %s", listener.Addr()) eshistory/eshistory_api/eshistory_api.go If logRequests is true, as noted earlier, (we wrap the handler in a request-logging handler)
go memstats.MemStats() } var router http.Handler log.Infof("logRequests: %s", *logRequests) if *logRequests { router = request_logger.LoggingHandler(os.Stdout, routes.NewRouter(app_routes)) } else { router = routes.NewRouter(app_routes) } httpServer := &http.Server{ Handler: router, // MaxHeaderBytes: MaxReqHeaderBytes, } log.Infof("INFO: Listening for requests on %s", listener.Addr()) eshistory/eshistory_api/eshistory_api.go This pattern is common in Go, and is analogous to Python decorators. Finally, we (tell the app to serve)
var app_routes = routes.Routes{ routes.Route{ "Ping", "GET", "/ping", PingHandler, }, routes.Route{ "Get", "GET", "/get", GetHandler, }, eshistory/eshistory_api/routes.go Each route has a name, a method, a URL pattern and a (HandlerFunc)
"github.com/gorilla/mux" ) // Route defines a route with a Name, Method, Pattern and HandlerFunc type Route struct { Name string Method string Pattern string HandlerFunc http.HandlerFunc } eshistory/eshistory_api/routes/routes.go In this subpackage we define what a Route is
NewRouter(routes Routes) *mux.Router { router := mux.NewRouter().StrictSlash(false) for _, route := range routes { router.Methods(route.Method) .Path(route.Pattern) .Name(route.Name) .Handler(route.HandlerFunc) } return router } eshistory/eshistory_api/routes/routes.go In eshistory_api.go, our main func passes those routes into the NewRouter function to create the net/http Handler. Moving on (to our main handler definitions)
"net/http" "sort" "strconv" "strings" "time" "bitly/eshistory/eshistory_api/api_formatter" "bitly/eshistory/internal/eshistory" "bitly/eshistory/internal/messages" "bitly/libbitly/settings" eshistory/eshistory_api/handlers.go we have our imports. I’d like to point out (api_formatter)
"net/http" "sort" "strconv" "strings" "time" "bitly/eshistory/eshistory_api/api_formatter" "bitly/eshistory/internal/eshistory" "bitly/eshistory/internal/messages" "bitly/libbitly/settings" eshistory/eshistory_api/handlers.go which is our Go library for formatting our conventional API JSON responses. The equivalent of the code in Python is in BaseHandler class.
"github.com/jehiah/memcache_pycompat" ) type tag struct { Value string `json:"tag"` } type tags []tag type userTagsResponse struct { eshistory/eshistory_api/handlers.go Finishing up the imports, I highly recommend (you use logrus)
"github.com/jehiah/memcache_pycompat" ) type tag struct { Value string `json:"tag"` } type tags []tag type userTagsResponse struct { eshistory/eshistory_api/handlers.go as it is a great tool that provides a massive improvement over standard library logging. You get log levels, structured data, output formats including text, XML and JSON, and TTY coloring. (Next we have)
tags `json:"results"` ResultCount int `json:"result_count"` } type searchRegexResponse struct { Page int `json:"page"` PerPage int `json:"perpage"` Results []eshistory.Link `json:"results"` ResultCount int `json:"result_count"` } type backfillPrivacyResponse struct { User string `json:"user"` eshistory/eshistory_api/handlers.go a set of structs used to establish types for responses from handlers. Useful pattern, define the output of handlers to ensure consistency and make it easy to fill in the blanks, and encode into JSON.
and their total count. func UserTagsHandler(rw http.ResponseWriter, req *http.Request) { w := &api_formatter.JSONResponseWriter{rw, false} user := req.FormValue("user") handlerName := "UserTagsHandler" if user == "bitly" { log.Warnf("query for bitly user should never happen %r", req) if err := w.WriteAPIResponse(nil); err != nil { log.WithFields(log.Fields{"error": err, "handler": handlerName}).Error("WriteAPIResponse error") } return } statsdQueue.Increment("tags") eshistory/eshistory_api/handlers.go Moving on to an actual handler implementation, notice (the use of JSONResponseWriter from api_formatter)
and their total count. func UserTagsHandler(rw http.ResponseWriter, req *http.Request) { w := &api_formatter.JSONResponseWriter{rw, false} user := req.FormValue("user") handlerName := "UserTagsHandler" if user == "bitly" { log.Warnf("query for bitly user should never happen %r", req) if err := w.WriteAPIResponse(nil); err != nil { log.WithFields(log.Fields{"error": err, "handler": handlerName}).Error("WriteAPIResponse error") } return } statsdQueue.Increment("tags") eshistory/eshistory_api/handlers.go Which is another wrapper, in this case of http.ResponseWriter. (Here’s an example of logrus)
and their total count. func UserTagsHandler(rw http.ResponseWriter, req *http.Request) { w := &api_formatter.JSONResponseWriter{rw, false} user := req.FormValue("user") handlerName := "UserTagsHandler" if user == "bitly" { log.Warnf("query for bitly user should never happen %r", req) if err := w.WriteAPIResponse(nil); err != nil { log.WithFields(log.Fields{"error": err, "handler": handlerName}).Error("WriteAPIResponse error") } return } statsdQueue.Increment("tags") tagFilter := req.FormValue("tag_filter") userTags, err := eshistoryClient.GetUserTags(user) if err != nil { eshistory/eshistory_api/handlers.go showing how you can add structured field data
and their total count. func UserTagsHandler(rw http.ResponseWriter, req *http.Request) { w := &api_formatter.JSONResponseWriter{rw, false} user := req.FormValue("user") handlerName := "UserTagsHandler" if user == "bitly" { log.Warnf("query for bitly user should never happen %r", req) if err := w.WriteAPIResponse(nil); err != nil { log.WithFields(log.Fields{"error": err, "handler": handlerName}).Error("WriteAPIResponse error") } return } statsdQueue.Increment("tags") tagFilter := req.FormValue("tag_filter") userTags, err := eshistoryClient.GetUserTags(user) if err != nil { eshistory/eshistory_api/handlers.go as well as specify the log level
range uTags { if strings.HasPrefix(t.Value, string(tagFilter)) { filteredTags = append(filteredTags, t) } } sort.Sort(filteredTags) finalResponse := userTagsResponse{ Results: filteredTags, ResultCount: len(filteredTags), } if err := w.WriteAPIResponse(finalResponse); err != nil { log.WithFields(log.Fields{"response": finalResponse, "error": err, "handler": handlerName}).Error("WriteAPIResponse error") } eshistory/eshistory_api/handlers.go I’ve skipped down a bit in UserTagsHandler, but here we have a list of tags that we filter, then sort (then populate a userTagsResponse struct)
range uTags { if strings.HasPrefix(t.Value, string(tagFilter)) { filteredTags = append(filteredTags, t) } } sort.Sort(filteredTags) finalResponse := userTagsResponse{ Results: filteredTags, ResultCount: len(filteredTags), } if err := w.WriteAPIResponse(finalResponse); err != nil { log.WithFields(log.Fields{"response": finalResponse, "error": err, "handler": handlerName}).Error("WriteAPIResponse error") } eshistory/eshistory_api/handlers.go to convert to our standard JSON response format. So (let’s take a look at the API response writer)
wrapping our responses as our particular JSON format package api_formatter import ( "bufio" "encoding/json" "net" "net/http" log "github.com/Sirupsen/logrus" ) // A ResponseWriter interface dedicated to JSON HTTP response. // Note that the object instantiated by the ResourceHandler that implements this interface, // also happens to implement http.ResponseWriter, http.Flusher and http.CloseNotifier. type ResponseWriter interface { eshistory/eshistory_api/api_formatter/api_response_writer.go Here we declare the standard net/http ResponseWriter interface, and implement the required methods — (Golang inheritance, effectively)
http.Header // Use EncodeJSON to generate the payload, write the headers with http.StatusOK if // they are not already written, then write the payload. // The Content-‐Type header is set to "application/json", unless already specified. WriteJSON(v interface{}) error // Encode the data structure to JSON, mainly used to wrap ResponseWriter in // middlewares. EncodeJSON(v interface{}) ([]byte, error) // Similar to the http.ResponseWriter interface, with additional JSON related // headers set. WriteHeader(int) // Use Error to generate an error message using Bitly standard API response format Error(int, string, interface{}) eshistory/eshistory_api/api_formatter/api_response_writer.go We have Header, WriterHeader, WriteJSON, EncodeJSON, (Error method declarations)
http.Header // Use EncodeJSON to generate the payload, write the headers with http.StatusOK if // they are not already written, then write the payload. // The Content-‐Type header is set to "application/json", unless already specified. WriteJSON(v interface{}) error // Encode the data structure to JSON, mainly used to wrap ResponseWriter in // middlewares. EncodeJSON(v interface{}) ([]byte, error) // Similar to the http.ResponseWriter interface, with additional JSON related // headers set. WriteHeader(int) // Use Error to generate an error message using Bitly standard API response format Error(int, string, interface{}) eshistory/eshistory_api/api_formatter/api_response_writer.go So let’s look at three that make up the main logic, (WriteJSON, Error, and WriteAPIResponse)
format WriteAPIResponse(interface{}) } // APIResponse holds the structure of a Bitly-‐standard API response type APIResponse struct { Data interface{} `json:"data"` StatusCode int `json:"status_code"` StatusText string `json:"status_txt"` } // JSONResponseWriter instantiated by the resource handler. // It implements the following interfaces: // ResponseWriter // http.ResponseWriter // http.Flusher // http.CloseNotifier eshistory/eshistory_api/api_formatter/api_response_writer.go Again, we (define the structure of an APIResponse)
format WriteAPIResponse(interface{}) } // APIResponse holds the structure of a Bitly-‐standard API response type APIResponse struct { Data interface{} `json:"data"` StatusCode int `json:"status_code"` StatusText string `json:"status_txt"` } // JSONResponseWriter instantiated by the resource handler. // It implements the following interfaces: // ResponseWriter // http.ResponseWriter // http.Flusher // http.CloseNotifier eshistory/eshistory_api/api_formatter/api_response_writer.go StatusCode 200 or 404, StatusText OK or NOT FOUND
bool } // WriteHeader writes the HTTP Content-‐Type Header as part of the response, // and defaults to application/json; charset=utf-‐8 if not overridden by // w.Header().Set(...) func (w *JSONResponseWriter) WriteHeader(code int) { if w.Header().Get("Content-‐Type") == "" { w.Header().Set("Content-‐Type", "application/json; charset=utf-‐8") } w.ResponseWriter.WriteHeader(code) w.WroteHeader = true } eshistory/eshistory_api/api_formatter/api_response_writer.go We create (the struct of the ResponseWriter wrapper)
bool } // WriteHeader writes the HTTP Content-‐Type Header as part of the response, // and defaults to application/json; charset=utf-‐8 if not overridden by // w.Header().Set(...) func (w *JSONResponseWriter) WriteHeader(code int) { if w.Header().Get("Content-‐Type") == "" { w.Header().Set("Content-‐Type", "application/json; charset=utf-‐8") } w.ResponseWriter.WriteHeader(code) w.WroteHeader = true } eshistory/eshistory_api/api_formatter/api_response_writer.go and implement (WriteHeader)
bool } // WriteHeader writes the HTTP Content-‐Type Header as part of the response, // and defaults to application/json; charset=utf-‐8 if not overridden by // w.Header().Set(...) func (w *JSONResponseWriter) WriteHeader(code int) { if w.Header().Get("Content-‐Type") == "" { w.Header().Set("Content-‐Type", "application/json; charset=utf-‐8") } w.ResponseWriter.WriteHeader(code) w.WroteHeader = true } eshistory/eshistory_api/api_formatter/api_response_writer.go which (sets the content type)
bool } // WriteHeader writes the HTTP Content-‐Type Header as part of the response, // and defaults to application/json; charset=utf-‐8 if not overridden by // w.Header().Set(...) func (w *JSONResponseWriter) WriteHeader(code int) { if w.Header().Get("Content-‐Type") == "" { w.Header().Set("Content-‐Type", "application/json; charset=utf-‐8") } w.ResponseWriter.WriteHeader(code) w.WroteHeader = true } eshistory/eshistory_api/api_formatter/api_response_writer.go to application/json if it has not yet been set manually. Next up is (WriteJSON)
// API error response, e.g. `{"status_code": <code>, "status_txt": "OH NOES", "data": ...}` func (w *JSONResponseWriter) Error(c int, t string, v interface{}) error { w.ResponseWriter.WriteHeader(c) w.WroteHeader = true r := APIResponse{ Data: v, StatusCode: c, StatusText: t, } err := w.WriteJSON(r) return err } eshistory/eshistory_api/api_formatter/api_response_writer.go and writes it out to the client.
queues allow for asynchronous data distribution. Examples include Apache Kafka and RabbitMQ. But before we get into the details, I’d like to (make a very important distinction)
include the main message queue server, nsqd, and other servers that round out the ecosystem: nsqlookupd for topic and channel lookup, nsqadmin for administration, and pynsqauthd for very basic authentication and authorization
Web App Database Worker Queue Web App Database Worker Queue composed of microservices listening for events published by other services. (How does a worker find the server or servers publishing to a particular topic?)
logatron_client.setup() Reader( topic='spam_api', channel='queuereader_spam_metrics', validate_method=validate_message, message_handler=count_spam_actions, lookupd_http_addresses=settings.get('nsq_lookupd') ) run() /<service>/queuereader_<service>.py When you are building a queuereader, this is how you find your topic. (So how are topics created?)
topic: 'spam_api' nsqlookupd topic: 'spam_api'? it asks nsqlookupd, which replies with the IP addresses of all nsqd instances publishing that topic. (So what is a channel?)
topic: 'spam_api' channel: 'spam_counter' When a queuereader connects to nsqd for a particular topic, it registers itself under a channel name, thereby creating it. (Channels are simply names for logical groups of consumers.)
But channels are important because messages are divided by # of subscribers to a channel; this allows horizontal scaling of queuereaders. (Each channel receives)
logatron_client.setup() Reader( topic=settings.get('nsqd_output_topic'), channel='queuereader_spam_metrics', validate_method=validate_message, message_handler=count_spam_actions, lookupd_http_addresses=settings.get('nsq_lookupd') ) run() /<service>/queuereader_<service>.py We looked at the topic, channel and nsqlookupd settings earlier, now I want to direct your attention to (validate method)
logatron_client.setup() Reader( topic=settings.get('nsqd_output_topic'), channel='queuereader_spam_metrics', validate_method=validate_message, message_handler=count_spam_actions, lookupd_http_addresses=settings.get('nsq_lookupd') ) run() /<service>/queuereader_<service>.py which determines whether a given message in the topic is one we want to handle or not. (If True)
and message.get('l'): return True if message.get('o') == '-‐' and message.get('l')\ and message.get('bl'): return True return False /<service>/queuereader_<service>.py we pass the message to the message handler function, if False we skip the message.
logatron_client.setup() Reader( topic=settings.get('nsqd_output_topic'), channel='queuereader_spam_metrics', validate_method=validate_message, message_handler=count_spam_actions, lookupd_http_addresses=settings.get('nsq_lookupd') ) run() /<service>/queuereader_<service>.py The message_handler setting specifies the function to call with the message.
key = key_section.get(message['l'], key_section['default']) statsd.incr(key) if key == 'remove_by_manual': key_section = statsd_keys['-‐manual'] key = key_section.get(message['bl'], key_section['default']) statsd.incr(key) return nsq_msg.finish() /<service>/queuereader_<service>.py which in this case just counts the number of spam encodes, and then (finishes)
key = key_section.get(message['l'], key_section['default']) statsd.incr(key) if key == 'remove_by_manual': key_section = statsd_keys['-‐manual'] key = key_section.get(message['bl'], key_section['default']) statsd.incr(key) return nsq_msg.finish() /<service>/queuereader_<service>.py the message. Calling finish() acks the message as handled to nsqd. Other options are to requeue the message, or requeue it with backoff.
"log" "sync" "bitly/eshistory/internal/eshistory" "bitly/eshistory/internal/statsd" "bitly/libbitly/exitchan" "bitly/libbitly/nsqutils" "bitly/libbitly/roles" "bitly/libbitly/settings" /eshistory/queuereader_bulk_action/queuereader_bulk_action.go We have our standard library imports and our Bitly and third-party imports
environment = flag.String("environment", "dev", "environment to execute in") settingsFile = flag.String("settings-‐file", "../ settings.json", "path to the settings.json file") ) type ActionHandler struct { Client *eshistory.Client /eshistory/queuereader_bulk_action/queuereader_bulk_action.go We set up the environment and settingsFile global variables
*eshistory.Client producer *nsq.Producer statsdQueue *statsdproxy.StatsdQueue } func main() { flag.Parse() settings.Load(*settingsFile, *environment) roles.LoadRoles("/bitly/local/conf/roles.json") // gpg = settings.OpenGpgSettingsFromSettingsFile(*settingsFile) /eshistory/queuereader_bulk_action/queuereader_bulk_action.go We create an ActionHandler with an elasticsearch client and a statsd connection. This ActionHandler is not only a consumer of nsqd messages, but also a producer — it writes messages to other topics.
err != nil { log.Fatalf("Failed creating producer %s", err) } var wg sync.WaitGroup wg.Add(1) /eshistory/queuereader_bulk_action/queuereader_bulk_action.go Here we set up the producer connection and the wait group.
"bulk_action", Channel: "queuereader_bulk_action", Handler: handler, MaxInFlight: settings.GetInt("max_in_flight"), ExitChan: exitchan.New(), WaitGroup: &wg, UserAgent: "queuereader_bulk_action", }) wg.Wait() } /eshistory/queuereader_bulk_action/queuereader_bulk_action.go Then we tie it all together by creating and running a consumer of the "bulk_action" topic. Our handler logic is implemented in (bulk_action_sq.go)
"log" "bitly/eshistory/internal/eshistory" "bitly/eshistory/internal/messages" "bitly/libbitly/settings" "github.com/bitly/go-‐nsq" "github.com/deckarep/golang-‐set" ) /eshistory/queuereader_bulk_action/bulk_action_sq.go Which starts out with imports
eshistory links, err := h.SearchLinks(msg.User, query) if err != nil { log.Printf("Error '%s' retrieving links for user %s with query %s", err, msg.User, query) return err } var action string var historyMsg messages.HistoryMessage for _, link := range links { /eshistory/queuereader_bulk_action/bulk_action_sq.go getting all the encodes from elasticsearch by calling SearchLinks
for _, link := range links { switch msg.Action { case "archive", "set_private": action = "edit" case "edit_tags": action = "edit_tags" // * Calculate new set of tags per link var iTags []interface{} if link.Tags != nil { iTags = toGenericSlice(*link.Tags) } /eshistory/queuereader_bulk_action/bulk_action_sq.go and for each encode, we create a message for the "history" topic with the appropriate action to take — tagging, archiving, or making the encode private.
tags per link, submit a new user_event message with link details if len(finalTags) > settings.GetInt("limit_tags_per_user") { if err = h.SendEventMessage(msg, *link.UserHash, startingTags); err != nil { log.Printf("Error sending event message: %s", err) return err } // * and skip to next link continue } default: continue /eshistory/queuereader_bulk_action/bulk_action_sq.go If tagging, we check for exceeding the limits of tags for the user, and submit an "event" message via SendEventMessage.
func (h *ActionHandler) SendEventMessage(b messages.BulkAction, uhash string, tags []string) error { userEventMsg := messages.UserEventMessage{ Action: "exceeded_limit_tags_per_user", Timestamp: b.Timestamp, CurrentUser: b.User, Link: uhash, AddTags: b.AddTags, RemoveTags: b.RemoveTags, StartingTags: tags, RemoteIP: b.RemoteIP, } /eshistory/queuereader_bulk_action/bulk_action_sq.go defines the helper method on the ActionHandler to submit a user event message to the 'event' topic. We build a new NSQ message
nil { log.Printf("Error marshalling user event message to JSON bytes: %#v", userEventMsg) return err } if err = h.producer.Publish("event", u); err != nil { log.Printf("Error publishing to nsq %s", err) return err } return nil } /eshistory/queuereader_bulk_action/bulk_action_sq.go marshal it to JSON, and publish it to nsqd. Next, let's take a look at the included quality of life (utilities)
messages go with it. You can tweak how much memory it is allowed to use, and messages over that mark get written to disk, but setting memory to zero is not a realistic strategy.
notice. When it comes back up, nsqlookupd will eventually notice. If the nsqlookupd instances are partitioned from each other, one can know about an nsqd instance the others do not. (Some notes and then questions)
tpherndon
isaoshimizu
hirosatogamo
tenshoku_draft
kamina_zzz
ryder472
negi111111
sadnessojisan
oracle4engineer
akexorcist
eijikominami
fujiwara3
miu_crescent
afnizarnur
colly
skipperchong
soudai
geeforr
sugarenia
jnunemaker
csswizardry
reverentgeek
jlugia
thoeni
sstephenson
![Peter Herndon [email protected] @tpherndon backend developer for Bitly, speaking on](https://files.speakerdeck.com/presentations/da531753a29c400e80056b726b50dc1b/slide_1.jpg){kind=link}
![def count_spam_actions(message, nsq_msg): key_section = statsd_keys[message['o']]](https://files.speakerdeck.com/presentations/da531753a29c400e80056b726b50dc1b/slide_156.jpg){kind=link}
![def count_spam_actions(message, nsq_msg): key_section = statsd_keys[message['o']]](https://files.speakerdeck.com/presentations/da531753a29c400e80056b726b50dc1b/slide_157.jpg){kind=link}
