From 9b94c47ee07b1728cbee979f23ae72236c64abe9 Mon Sep 17 00:00:00 2001 From: Paul Ramsey Date: Wed, 21 Apr 2021 16:08:21 -0700 Subject: [PATCH] Put global map access within mutexes to try to address #89 --- assets/index.html | 4 +- assets/preview-function.html | 8 +-- assets/preview-table.html | 8 +-- db.go | 16 ++--- layer.go | 66 +++++++++---------- layer_proc.go | 54 +++++++++------- layer_table.go | 122 +++++++++++++++++------------------ main.go | 75 +++++++++++---------- main_test.go | 12 ++-- tile.go | 2 +- util.go | 21 +++--- 11 files changed, 202 insertions(+), 186 deletions(-) diff --git a/assets/index.html b/assets/index.html index 2461dc3..e5e5789 100644 --- a/assets/index.html +++ b/assets/index.html @@ -23,7 +23,7 @@

Table Layers

{{ range $key, $value := . }} {{ if eq $value.Type "table" }}
  • - {{ $value.Id }} (preview | json) + {{ $value.ID }} (preview | json) {{ if $value.Description }}
    {{ $value.Description }} {{ end }}
    • {{ end }} @@ -35,7 +35,7 @@

    Function Layers

    {{ range $key, $value := . }} {{ if eq $value.Type "function" }}
  • - {{ $value.Id }} (preview | json) + {{ $value.ID }} (preview | json) {{ if $value.Description }}
    {{ $value.Description }} {{ end }}
    • {{ end }} diff --git a/assets/preview-function.html b/assets/preview-function.html index d19e41b..8ddb613 100644 --- a/assets/preview-function.html +++ b/assets/preview-function.html @@ -1,7 +1,7 @@ - pg_tileserv: {{.Id}} + pg_tileserv: {{.ID}} @@ -15,7 +15,7 @@
    -

    {{ .Id }}

    +

    {{ .ID }}

    {{ if .Description }}

    {{ .Description }}

    {{ end }} @@ -44,9 +44,9 @@

    {{ .Id }}

    var map; var mapcolor = "blue"; - $.getJSON("{{ .Id }}.json", function(layer) { + $.getJSON("{{ .ID }}.json", function(layer) { // A little info for explorers - console.log("{{ .Id }}.json"); + console.log("{{ .ID }}.json"); console.log(layer); layer["arguments"].forEach(function (item, i) { diff --git a/assets/preview-table.html b/assets/preview-table.html index 4955296..83f2ee7 100644 --- a/assets/preview-table.html +++ b/assets/preview-table.html @@ -1,7 +1,7 @@ - pg_tileserv: {{.Id}} + pg_tileserv: {{.ID}} @@ -16,7 +16,7 @@
    -

    {{ .Id }}

    +

    {{ .ID }}

    {{ if .Description }}

    {{ .Description }}

    {{ end }} @@ -32,9 +32,9 @@

    {{ .Id }}

    var map; var mapcolor = "blue"; - $.getJSON("{{ .Id }}.json", function(layer) { + $.getJSON("{{ .ID }}.json", function(layer) { // A little info for explorers - console.log("{{ .Id }}.json"); + console.log("{{ .ID }}.json"); console.log(layer); var mapConfig = { diff --git a/db.go b/db.go index c7218a2..4f910b9 100644 --- a/db.go +++ b/db.go @@ -22,7 +22,7 @@ import ( log "github.com/sirupsen/logrus" ) -func DbConnect() (*pgxpool.Pool, error) { +func dbConnect() (*pgxpool.Pool, error) { if globalDb == nil { var err error var config *pgxpool.Config @@ -67,8 +67,8 @@ func DbConnect() (*pgxpool.Pool, error) { return globalDb, nil } -func LoadVersions() error { - db, err := DbConnect() +func loadVersions() error { + db, err := dbConnect() if err != nil { return err } @@ -105,13 +105,13 @@ func LoadVersions() error { return nil } -func DBTileRequest(ctx context.Context, tr *TileRequest) ([]byte, error) { - db, err := DbConnect() +func dBTileRequest(ctx context.Context, tr *TileRequest) ([]byte, error) { + db, err := dbConnect() if err != nil { log.Error(err) return nil, err } - row := db.QueryRow(ctx, tr.Sql, tr.Args...) + row := db.QueryRow(ctx, tr.SQL, tr.Args...) var mvtTile []byte err = row.Scan(&mvtTile) if err != nil { @@ -123,13 +123,13 @@ func DBTileRequest(ctx context.Context, tr *TileRequest) ([]byte, error) { if pgconn.Timeout(err) { return nil, tileAppError{ SrcErr: err, - Message: fmt.Sprintf("Timeout: deadline exceeded on %s/%s", tr.LayerId, tr.Tile.String()), + Message: fmt.Sprintf("Timeout: deadline exceeded on %s/%s", tr.LayerID, tr.Tile.String()), } } return nil, tileAppError{ SrcErr: err, - Message: fmt.Sprintf("SQL error on %s/%s", tr.LayerId, tr.Tile.String()), + Message: fmt.Sprintf("SQL error on %s/%s", tr.LayerID, tr.Tile.String()), } } diff --git a/layer.go b/layer.go index f3734be..c633d6c 100644 --- a/layer.go +++ b/layer.go @@ -7,18 +7,21 @@ import ( "net/http" ) -type layerType int +// LayerType is the table/function type of a layer +type LayerType int const ( - layerTypeTable = 1 - layerTypeFunction = 2 + // LayerTypeTable is a table layer + LayerTypeTable = 1 + // LayerTypeFunction is a function layer + LayerTypeFunction = 2 ) -func (lt layerType) String() string { +func (lt LayerType) String() string { switch lt { - case layerTypeTable: + case LayerTypeTable: return "table" - case layerTypeFunction: + case LayerTypeFunction: return "function" default: return "unknown" @@ -30,80 +33,77 @@ func (lt layerType) String() string { // a TileRequest containing SQL to produce tiles // given an input tile type Layer interface { - GetType() layerType - GetId() string + GetType() LayerType + GetID() string GetDescription() string GetName() string GetSchema() string GetTileRequest(tile Tile, r *http.Request) TileRequest - WriteLayerJson(w http.ResponseWriter, req *http.Request) error + WriteLayerJSON(w http.ResponseWriter, req *http.Request) error } type TileRequest struct { - LayerId string + LayerID string Tile Tile - Sql string + SQL string Args []interface{} } -// A global array of Layer where the state is held for performance -// Refreshed when LoadLayerTableList is called -// Key is of the form: schemaname.tablename -var globalLayers map[string]Layer - -func GetLayer(lyrId string) (Layer, error) { - if lyr, ok := globalLayers[lyrId]; ok { +func getLayer(lyrID string) (Layer, error) { + lyr, ok := globalLayers[lyrID] + if ok { return lyr, nil - } else { - return lyr, errors.New(fmt.Sprintf("Unable to get layer '%s'", lyrId)) } + return lyr, errors.New(fmt.Sprintf("Unable to get layer '%s'", lyrID)) } -func LoadLayers() error { +func loadLayers() error { _, errBnd := getServerBounds() if errBnd != nil { return errBnd } - tableLayers, errTl := GetTableLayers() + tableLayers, errTl := getTableLayers() if errTl != nil { return errTl } - functionLayers, errFl := GetFunctionLayers() + functionLayers, errFl := getFunctionLayers() if errFl != nil { return errFl } // Empty the global layer map + globalLayersMutex.Lock() globalLayers = make(map[string]Layer) for _, lyr := range tableLayers { - globalLayers[lyr.GetId()] = lyr + globalLayers[lyr.GetID()] = lyr } for _, lyr := range functionLayers { - globalLayers[lyr.GetId()] = lyr + globalLayers[lyr.GetID()] = lyr } + globalLayersMutex.Unlock() return nil } -type LayerJson struct { +type layerJSON struct { Type string `json:"type"` - Id string `json:"id"` + ID string `json:"id"` Name string `json:"name"` Schema string `json:"schema"` Description string `json:"description"` - DetailUrl string `json:"detailurl"` + DetailURL string `json:"detailurl"` } -func GetJsonLayers(r *http.Request) map[string]LayerJson { - json := make(map[string]LayerJson) +func getJSONLayers(r *http.Request) map[string]layerJSON { + json := make(map[string]layerJSON) urlBase := serverURLBase(r) for k, v := range globalLayers { - json[k] = LayerJson{ + json[k] = layerJSON{ Type: v.GetType().String(), - Id: v.GetId(), + ID: v.GetID(), Name: v.GetName(), Schema: v.GetSchema(), Description: v.GetDescription(), - DetailUrl: fmt.Sprintf("%s/%s.json", urlBase, v.GetId()), + DetailURL: fmt.Sprintf("%s/%s.json", urlBase, v.GetID()), } } return json diff --git a/layer_proc.go b/layer_proc.go index a35a95d..7e84082 100644 --- a/layer_proc.go +++ b/layer_proc.go @@ -16,9 +16,9 @@ import ( "github.com/spf13/viper" ) -// type LayerTable struct { +// LayerFunction provides metadata about the function type LayerFunction struct { - Id string + ID string Schema string Function string Description string @@ -28,6 +28,8 @@ type LayerFunction struct { Tiles string } +// FunctionArgument provides the metadata and order +// of arguments in function call. type FunctionArgument struct { Name string `json:"name"` Type string `json:"type"` @@ -35,27 +37,29 @@ type FunctionArgument struct { order int } -type FunctionDetailJson struct { - Id string `json:"id"` +// FunctionDetailJSON gives the output structure for +// the function. +type FunctionDetailJSON struct { + ID string `json:"id"` Schema string `json:"schema"` Name string `json:"name"` Description string `json:"description,omitempty"` Arguments []FunctionArgument `json:"arguments"` MinZoom int `json:"minzoom"` MaxZoom int `json:"maxzoom"` - TileUrl string `json:"tileurl"` + TileURL string `json:"tileurl"` } /******************************************************************************** * Layer Interface */ -func (lyr LayerFunction) GetType() layerType { - return layerTypeFunction +func (lyr LayerFunction) GetType() LayerType { + return LayerTypeFunction } -func (lyr LayerFunction) GetId() string { - return lyr.Id +func (lyr LayerFunction) GetID() string { + return lyr.ID } func (lyr LayerFunction) GetDescription() string { @@ -73,19 +77,19 @@ func (lyr LayerFunction) GetSchema() string { func (lyr LayerFunction) GetTileRequest(tile Tile, r *http.Request) TileRequest { procArgs := lyr.getFunctionArgs(r.URL.Query()) - sql, data, _ := lyr.requestSql(tile, procArgs) + sql, data, _ := lyr.requestSQL(tile, procArgs) tr := TileRequest{ - LayerId: lyr.Id, + LayerID: lyr.ID, Tile: tile, - Sql: sql, + SQL: sql, Args: data, } return tr } -func (lyr LayerFunction) WriteLayerJson(w http.ResponseWriter, req *http.Request) error { - jsonTableDetail, err := lyr.getFunctionDetailJson(req) +func (lyr LayerFunction) WriteLayerJSON(w http.ResponseWriter, req *http.Request) error { + jsonTableDetail, err := lyr.getFunctionDetailJSON(req) if err != nil { return err } @@ -97,7 +101,7 @@ func (lyr LayerFunction) WriteLayerJson(w http.ResponseWriter, req *http.Request /********************************************************************************/ -func (lyr *LayerFunction) requestSql(tile Tile, args map[string]string) (string, []interface{}, error) { +func (lyr *LayerFunction) requestSQL(tile Tile, args map[string]string) (string, []interface{}, error) { // Need ordered list of named parameters and values to // pass into the Query keys := []string{"x => $1", "y => $2", "z => $3"} @@ -106,7 +110,7 @@ func (lyr *LayerFunction) requestSql(tile Tile, args map[string]string) (string, for k, v := range args { keys = append(keys, fmt.Sprintf("%s => $%d", k, i)) vals = append(vals, v) - i += 1 + i++ } // Build the SQL @@ -136,10 +140,10 @@ func (lyr *LayerFunction) getFunctionArgs(vals url.Values) map[string]string { return funcArgs } -func (lyr *LayerFunction) getFunctionDetailJson(req *http.Request) (FunctionDetailJson, error) { +func (lyr *LayerFunction) getFunctionDetailJSON(req *http.Request) (FunctionDetailJSON, error) { - td := FunctionDetailJson{ - Id: lyr.Id, + td := FunctionDetailJSON{ + ID: lyr.ID, Schema: lyr.Schema, Name: lyr.Function, Description: lyr.Description, @@ -148,7 +152,7 @@ func (lyr *LayerFunction) getFunctionDetailJson(req *http.Request) (FunctionDeta MaxZoom: viper.GetInt("DefaultMaxZoom"), } // TileURL is relative to server base - td.TileUrl = fmt.Sprintf("%s/%s/{z}/{x}/{y}.pbf", serverURLBase(req), lyr.Id) + td.TileURL = fmt.Sprintf("%s/%s/{z}/{x}/{y}.pbf", serverURLBase(req), lyr.ID) tmpMap := make(map[int]FunctionArgument) tmpKeys := make([]int, 0, len(lyr.Arguments)) @@ -163,11 +167,11 @@ func (lyr *LayerFunction) getFunctionDetailJson(req *http.Request) (FunctionDeta return td, nil } -func GetFunctionLayers() ([]LayerFunction, error) { +func getFunctionLayers() ([]LayerFunction, error) { // Valid functions **must** have signature of // function(z integer, x integer, y integer) returns bytea - layerSql := ` + layerSQL := ` SELECT Format('%s.%s', n.nspname, p.proname) AS id, n.nspname, @@ -186,12 +190,12 @@ func GetFunctionLayers() ([]LayerFunction, error) { ORDER BY 1 ` - db, connerr := DbConnect() + db, connerr := dbConnect() if connerr != nil { return nil, connerr } - rows, err := db.Query(context.Background(), layerSql) + rows, err := db.Query(context.Background(), layerSQL) if err != nil { return nil, err } @@ -228,7 +232,7 @@ func GetFunctionLayers() ([]LayerFunction, error) { } lyr := LayerFunction{ - Id: id, + ID: id, Schema: schema, Function: function, Description: description, diff --git a/layer_table.go b/layer_table.go index 929ede7..98e1044 100644 --- a/layer_table.go +++ b/layer_table.go @@ -21,13 +21,13 @@ import ( ) type LayerTable struct { - Id string + ID string Schema string Table string Description string Properties map[string]TableProperty GeometryType string - IdColumn string + IDColumn string GeometryColumn string Srid int } @@ -39,8 +39,8 @@ type TableProperty struct { order int } -type TableDetailJson struct { - Id string `json:"id"` +type TableDetailJSON struct { + ID string `json:"id"` Schema string `json:"schema"` Name string `json:"name"` Description string `json:"description,omitempty"` @@ -50,19 +50,19 @@ type TableDetailJson struct { Bounds [4]float64 `json:"bounds"` MinZoom int `json:"minzoom"` MaxZoom int `json:"maxzoom"` - TileUrl string `json:"tileurl"` + TileURL string `json:"tileurl"` } /******************************************************************************** * Layer Interface */ -func (lyr LayerTable) GetType() layerType { - return layerTypeTable +func (lyr LayerTable) GetType() LayerType { + return LayerTypeTable } -func (lyr LayerTable) GetId() string { - return lyr.Id +func (lyr LayerTable) GetID() string { + return lyr.ID } func (lyr LayerTable) GetDescription() string { @@ -77,8 +77,8 @@ func (lyr LayerTable) GetSchema() string { return lyr.Schema } -func (lyr LayerTable) WriteLayerJson(w http.ResponseWriter, req *http.Request) error { - jsonTableDetail, err := lyr.getTableDetailJson(req) +func (lyr LayerTable) WriteLayerJSON(w http.ResponseWriter, req *http.Request) error { + jsonTableDetail, err := lyr.getTableDetailJSON(req) if err != nil { return err } @@ -90,12 +90,12 @@ func (lyr LayerTable) WriteLayerJson(w http.ResponseWriter, req *http.Request) e func (lyr LayerTable) GetTileRequest(tile Tile, r *http.Request) TileRequest { rp := lyr.getQueryParameters(r.URL.Query()) - sql, _ := lyr.requestSql(&tile, &rp) + sql, _ := lyr.requestSQL(&tile, &rp) tr := TileRequest{ - LayerId: lyr.Id, + LayerID: lyr.ID, Tile: tile, - Sql: sql, + SQL: sql, Args: nil, } return tr @@ -150,7 +150,7 @@ func (lyr *LayerTable) getQueryPropertiesParameter(q url.Values) []string { lyrAtts := (*lyr).Properties queryAtts := make([]string, 0, len(lyrAtts)) - haveIdColumn := false + haveIDColumn := false if haveProperties { aAtts := strings.Split(sAtts[0], ",") @@ -160,8 +160,8 @@ func (lyr *LayerTable) getQueryPropertiesParameter(q url.Values) []string { decAtt = strings.Trim(decAtt, " ") att, ok := lyrAtts[decAtt] if ok { - if att.Name == lyr.IdColumn { - haveIdColumn = true + if att.Name == lyr.IDColumn { + haveIDColumn = true } queryAtts = append(queryAtts, att.Name) } @@ -175,8 +175,8 @@ func (lyr *LayerTable) getQueryPropertiesParameter(q url.Values) []string { queryAtts = append(queryAtts, v.Name) } } - if (!haveIdColumn) && lyr.IdColumn != "" { - queryAtts = append(queryAtts, lyr.IdColumn) + if (!haveIDColumn) && lyr.IDColumn != "" { + queryAtts = append(queryAtts, lyr.IDColumn) } return queryAtts } @@ -205,9 +205,9 @@ func (lyr *LayerTable) getQueryParameters(q url.Values) queryParameters { /********************************************************************************/ -func (lyr *LayerTable) getTableDetailJson(req *http.Request) (TableDetailJson, error) { - td := TableDetailJson{ - Id: lyr.Id, +func (lyr *LayerTable) getTableDetailJSON(req *http.Request) (TableDetailJSON, error) { + td := TableDetailJSON{ + ID: lyr.ID, Schema: lyr.Schema, Name: lyr.Table, Description: lyr.Description, @@ -216,7 +216,7 @@ func (lyr *LayerTable) getTableDetailJson(req *http.Request) (TableDetailJson, e MaxZoom: viper.GetInt("DefaultMaxZoom"), } // TileURL is relative to server base - td.TileUrl = fmt.Sprintf("%s/%s/{z}/{x}/{y}.pbf", serverURLBase(req), lyr.Id) + td.TileURL = fmt.Sprintf("%s/%s/{z}/{x}/{y}.pbf", serverURLBase(req), lyr.ID) // Want to add the properties to the Json representation // in table order, which is fiddly @@ -248,7 +248,7 @@ func (lyr *LayerTable) getTableDetailJson(req *http.Request) (TableDetailJson, e func (lyr *LayerTable) GetBoundsExact() (Bounds, error) { bounds := Bounds{} - extentSql := fmt.Sprintf(` + extentSQL := fmt.Sprintf(` WITH ext AS ( SELECT coalesce( @@ -265,7 +265,7 @@ func (lyr *LayerTable) GetBoundsExact() (Bounds, error) { FROM ext `, lyr.GeometryColumn, lyr.Srid, lyr.Schema, lyr.Table) - db, err := DbConnect() + db, err := dbConnect() if err != nil { return bounds, err } @@ -275,7 +275,7 @@ func (lyr *LayerTable) GetBoundsExact() (Bounds, error) { ymin pgtype.Float8 ymax pgtype.Float8 ) - err = db.QueryRow(context.Background(), extentSql).Scan(&xmin, &ymin, &xmax, &ymax) + err = db.QueryRow(context.Background(), extentSQL).Scan(&xmin, &ymin, &xmax, &ymax) if err != nil { return bounds, tileAppError{ SrcErr: err, @@ -294,7 +294,7 @@ func (lyr *LayerTable) GetBoundsExact() (Bounds, error) { func (lyr *LayerTable) GetBounds() (Bounds, error) { bounds := Bounds{} - extentSql := fmt.Sprintf(` + extentSQL := fmt.Sprintf(` WITH ext AS ( SELECT ST_Transform(ST_SetSRID(ST_EstimatedExtent('%s', '%s', '%s'), %d), 4326) AS geom ) @@ -306,7 +306,7 @@ func (lyr *LayerTable) GetBounds() (Bounds, error) { FROM ext `, lyr.Schema, lyr.Table, lyr.GeometryColumn, lyr.Srid) - db, err := DbConnect() + db, err := dbConnect() if err != nil { return bounds, err } @@ -317,7 +317,7 @@ func (lyr *LayerTable) GetBounds() (Bounds, error) { ymin pgtype.Float8 ymax pgtype.Float8 ) - err = db.QueryRow(context.Background(), extentSql).Scan(&xmin, &ymin, &xmax, &ymax) + err = db.QueryRow(context.Background(), extentSQL).Scan(&xmin, &ymin, &xmax, &ymax) if err != nil { return bounds, tileAppError{ SrcErr: err, @@ -345,11 +345,11 @@ func (lyr *LayerTable) GetBounds() (Bounds, error) { return bounds, nil } -func (lyr *LayerTable) requestSql(tile *Tile, qp *queryParameters) (string, error) { +func (lyr *LayerTable) requestSQL(tile *Tile, qp *queryParameters) (string, error) { type sqlParameters struct { - TileSql string - QuerySql string + TileSQL string + QuerySQL string TileSrid int Resolution int Buffer int @@ -366,9 +366,9 @@ func (lyr *LayerTable) requestSql(tile *Tile, qp *queryParameters) (string, erro // expanded version for querying tileBounds := tile.Bounds queryBounds := tile.Bounds - queryBounds.Expand(tile.Width() * float64(qp.Buffer) / float64(qp.Resolution)) - tileSql := tileBounds.SQL() - tileQuerySql := queryBounds.SQL() + queryBounds.Expand(tile.width() * float64(qp.Buffer) / float64(qp.Resolution)) + tileSQL := tileBounds.SQL() + tileQuerySQL := queryBounds.SQL() // SRID of the tile we are going to generate, which might be different // from the layer SRID in the database @@ -383,18 +383,18 @@ func (lyr *LayerTable) requestSql(tile *Tile, qp *queryParameters) (string, erro // only specify MVT format parameters we have configured mvtParams := make([]string, 0) - mvtParams = append(mvtParams, fmt.Sprintf("'%s', %d", lyr.Id, qp.Resolution)) + mvtParams = append(mvtParams, fmt.Sprintf("'%s', %d", lyr.ID, qp.Resolution)) if lyr.GeometryColumn != "" { mvtParams = append(mvtParams, fmt.Sprintf("'%s'", lyr.GeometryColumn)) } // The idColumn parameter is PostGIS3+ only - if globalPostGISVersion >= 3000000 && lyr.IdColumn != "" { - mvtParams = append(mvtParams, fmt.Sprintf("'%s'", lyr.IdColumn)) + if globalPostGISVersion >= 3000000 && lyr.IDColumn != "" { + mvtParams = append(mvtParams, fmt.Sprintf("'%s'", lyr.IDColumn)) } sp := sqlParameters{ - TileSql: tileSql, - QuerySql: tileQuerySql, + TileSQL: tileSQL, + QuerySQL: tileQuerySQL, TileSrid: tileSrid, Resolution: qp.Resolution, Buffer: qp.Buffer, @@ -412,7 +412,7 @@ func (lyr *LayerTable) requestSql(tile *Tile, qp *queryParameters) (string, erro // TODO: Remove ST_Force2D when fixes to line clipping are common // in GEOS. See https://trac.osgeo.org/postgis/ticket/4690 - tmplSql := ` + tmplSQL := ` SELECT ST_AsMVT(mvtgeom, {{ .MvtParams }}) FROM ( SELECT ST_AsMVTGeom( ST_Transform(ST_Force2D(t."{{ .GeometryColumn }}"), {{ .TileSrid }}), @@ -424,8 +424,8 @@ func (lyr *LayerTable) requestSql(tile *Tile, qp *queryParameters) (string, erro , {{ .Properties }} {{ end }} FROM "{{ .Schema }}"."{{ .Table }}" t, ( - SELECT {{ .TileSql }} AS geom_clip, - {{ .QuerySql }} AS geom_query + SELECT {{ .TileSQL }} AS geom_clip, + {{ .QuerySQL }} AS geom_query ) bounds WHERE ST_Intersects(t."{{ .GeometryColumn }}", ST_Transform(bounds.geom_query, {{ .Srid }})) @@ -433,16 +433,16 @@ func (lyr *LayerTable) requestSql(tile *Tile, qp *queryParameters) (string, erro ) mvtgeom ` - sql, err := renderSqlTemplate("tableTileSql", tmplSql, sp) + sql, err := renderSQLTemplate("tabletilesql", tmplSQL, sp) if err != nil { return "", err } return sql, err } -func GetTableLayers() ([]LayerTable, error) { +func getTableLayers() ([]LayerTable, error) { - layerSql := ` + layerSQL := ` SELECT Format('%s.%s', n.nspname, c.relname) AS id, n.nspname AS schema, @@ -478,12 +478,12 @@ func GetTableLayers() ([]LayerTable, error) { ORDER BY 1 ` - db, connerr := DbConnect() + db, connerr := dbConnect() if connerr != nil { return nil, connerr } - rows, err := db.Query(context.Background(), layerSql) + rows, err := db.Query(context.Background(), layerSQL) if err != nil { return nil, connerr } @@ -493,14 +493,14 @@ func GetTableLayers() ([]LayerTable, error) { for rows.Next() { var ( - id, schema, table, description, geometry_column string - srid int - geometry_type, id_column string - atts pgtype.TextArray + id, schema, table, description, geometryColumn string + srid int + geometryType, idColumn string + atts pgtype.TextArray ) - err := rows.Scan(&id, &schema, &table, &description, &geometry_column, - &srid, &geometry_type, &id_column, &atts) + err := rows.Scan(&id, &schema, &table, &description, &geometryColumn, + &srid, &geometryType, &idColumn, &atts) if err != nil { return nil, err } @@ -519,27 +519,27 @@ func GetTableLayers() ([]LayerTable, error) { elmLen := atts.Dimensions[1].Length for i := arrStart; i < arrLen; i++ { pos := i * elmLen - elmId := atts.Elements[pos].String + elmID := atts.Elements[pos].String elm := TableProperty{ - Name: elmId, + Name: elmID, Type: atts.Elements[pos+1].String, Description: atts.Elements[pos+2].String, } elm.order, _ = strconv.Atoi(atts.Elements[pos+3].String) - properties[elmId] = elm + properties[elmID] = elm } } // "schema.tablename" is our unique key for table layers lyr := LayerTable{ - Id: id, + ID: id, Schema: schema, Table: table, Description: description, - GeometryColumn: geometry_column, + GeometryColumn: geometryColumn, Srid: srid, - GeometryType: geometry_type, - IdColumn: id_column, + GeometryType: geometryType, + IDColumn: idColumn, Properties: properties, } diff --git a/main.go b/main.go index a0505a6..1cb8f09 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,7 @@ import ( "os" "os/signal" "strings" + "sync" "time" // REST routing @@ -56,6 +57,12 @@ var globalServerBounds *Bounds = nil // in the response headers var globalTimeToLive = -1 +// A global array of Layer where the state is held for performance +// Refreshed when LoadLayerTableList is called +// Key is of the form: schemaname.tablename +var globalLayers map[string]Layer +var globalLayersMutex = &sync.Mutex{} + /******************************************************************************/ func init() { @@ -132,8 +139,8 @@ func main() { log.Info("Run with --help parameter for commandline options") // Read environment configuration first - if dbUrl := os.Getenv("DATABASE_URL"); dbUrl != "" { - viper.Set("DbConnection", dbUrl) + if dbURL := os.Getenv("DATABASE_URL"); dbURL != "" { + viper.Set("DbConnection", dbURL) log.Info("Using database connection info from environment variable DATABASE_URL") } @@ -165,13 +172,13 @@ func main() { // Load the global layer list right away // Also connects to database - if err := LoadLayers(); err != nil { + if err := loadLayers(); err != nil { log.Fatal(err) } // Read the postgis_full_version string and store // in a global for version testing - if errv := LoadVersions(); errv != nil { + if errv := loadVersions(); errv != nil { log.Fatal(errv) } log.WithFields(log.Fields{ @@ -190,12 +197,12 @@ func main() { /******************************************************************************/ func requestPreview(w http.ResponseWriter, r *http.Request) error { - lyrId := mux.Vars(r)["name"] + lyrID := mux.Vars(r)["name"] log.WithFields(log.Fields{ "event": "request", "topic": "layerpreview", - "key": lyrId, - }).Tracef("requestPreview: %s", lyrId) + "key": lyrID, + }).Tracef("requestPreview: %s", lyrID) // reqProperties := r.FormValue("properties") // reqLimit := r.FormValue("limit") @@ -203,11 +210,11 @@ func requestPreview(w http.ResponseWriter, r *http.Request) error { // reqBuffer := r.FormValue("buffer") // Refresh the layers list - if err := LoadLayers(); err != nil { + if err := loadLayers(); err != nil { return err } // Get the requested layer - lyr, errLyr := GetLayer(lyrId) + lyr, errLyr := getLayer(lyrID) if errLyr != nil { return errLyr } @@ -233,17 +240,17 @@ func requestPreview(w http.ResponseWriter, r *http.Request) error { return nil } -func requestListHtml(w http.ResponseWriter, r *http.Request) error { +func requestListHTML(w http.ResponseWriter, r *http.Request) error { log.WithFields(log.Fields{ "event": "request", "topic": "layerlist", }).Trace("requestListHtml") // Update the global in-memory list from // the database - if err := LoadLayers(); err != nil { + if err := loadLayers(); err != nil { return err } - jsonLayers := GetJsonLayers(r) + jsonLayers := getJSONLayers(r) content, err := ioutil.ReadFile(fmt.Sprintf("%s/index.html", viper.GetString("AssetsPath"))) @@ -260,40 +267,40 @@ func requestListHtml(w http.ResponseWriter, r *http.Request) error { return nil } -func requestListJson(w http.ResponseWriter, r *http.Request) error { +func requestListJSON(w http.ResponseWriter, r *http.Request) error { log.WithFields(log.Fields{ "event": "request", "topic": "layerlist", }).Trace("requestListJson") // Update the global in-memory list from // the database - if err := LoadLayers(); err != nil { + if err := loadLayers(); err != nil { return err } w.Header().Add("Content-Type", "application/json") - jsonLayers := GetJsonLayers(r) + jsonLayers := getJSONLayers(r) json.NewEncoder(w).Encode(jsonLayers) return nil } -func requestDetailJson(w http.ResponseWriter, r *http.Request) error { - lyrId := mux.Vars(r)["name"] +func requestDetailJSON(w http.ResponseWriter, r *http.Request) error { + lyrID := mux.Vars(r)["name"] log.WithFields(log.Fields{ "event": "request", "topic": "layerdetail", - }).Tracef("requestDetailJson(%s)", lyrId) + }).Tracef("requestDetailJson(%s)", lyrID) // Refresh the layers list - if err := LoadLayers(); err != nil { + if err := loadLayers(); err != nil { return err } - lyr, errLyr := GetLayer(lyrId) + lyr, errLyr := getLayer(lyrID) if errLyr != nil { return errLyr } - errWrite := lyr.WriteLayerJson(w, r) + errWrite := lyr.WriteLayerJSON(w, r) if errWrite != nil { return errWrite } @@ -302,7 +309,7 @@ func requestDetailJson(w http.ResponseWriter, r *http.Request) error { func requestTile(w http.ResponseWriter, r *http.Request) error { vars := mux.Vars(r) - lyr, errLyr := GetLayer(vars["name"]) + lyr, errLyr := getLayer(vars["name"]) if errLyr != nil { return errLyr } @@ -315,13 +322,13 @@ func requestTile(w http.ResponseWriter, r *http.Request) error { "event": "request", "topic": "tile", "key": tile.String(), - }).Tracef("RequestLayerTile: %s", tile.String()) + }).Tracef("requestTile: %s", tile.String()) ctx, cancel := context.WithTimeout(context.Background(), viper.GetDuration("DbTimeout")*time.Second) defer cancel() tilerequest := lyr.GetTileRequest(tile, r) - mvt, errMvt := DBTileRequest(ctx, &tilerequest) + mvt, errMvt := dBTileRequest(ctx, &tilerequest) if errMvt != nil { return errMvt } @@ -341,7 +348,7 @@ func requestTile(w http.ResponseWriter, r *http.Request) error { // if they want to specify the particular HTTP error code to be used // in their error return type tileAppError struct { - HttpCode int + HTTPCode int SrcErr error Topic string Message string @@ -375,8 +382,8 @@ func (fn tileAppHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.WithField("correlation-id", hdr[0]) } if e, ok := err.(tileAppError); ok { - if e.HttpCode == 0 { - e.HttpCode = 500 + if e.HTTPCode == 0 { + e.HTTPCode = 500 } if e.Topic != "" { log.WithField("topic", e.Topic) @@ -384,7 +391,7 @@ func (fn tileAppHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.WithField("key", e.Message) log.WithField("src", e.SrcErr.Error()) log.Error(err) - http.Error(w, e.Error(), e.HttpCode) + http.Error(w, e.Error(), e.HTTPCode) } else { log.Error(err) http.Error(w, err.Error(), 500) @@ -407,7 +414,7 @@ func setCacheControl(next http.Handler) http.Handler { /******************************************************************************/ -func TileRouter() *mux.Router { +func tileRouter() *mux.Router { // creates a new instance of a mux router r := mux.NewRouter(). StrictSlash(true). @@ -418,12 +425,12 @@ func TileRouter() *mux.Router { Subrouter() // Front page and layer list - r.Handle("/", tileAppHandler(requestListHtml)) - r.Handle("/index.html", tileAppHandler(requestListHtml)) - r.Handle("/index.json", tileAppHandler(requestListJson)) + r.Handle("/", tileAppHandler(requestListHTML)) + r.Handle("/index.html", tileAppHandler(requestListHTML)) + r.Handle("/index.json", tileAppHandler(requestListJSON)) // Layer detail and demo pages r.Handle("/{name}.html", tileAppHandler(requestPreview)) - r.Handle("/{name}.json", tileAppHandler(requestDetailJson)) + r.Handle("/{name}.json", tileAppHandler(requestDetailJSON)) // Tile requests r.Handle("/{name}/{z:[0-9]+}/{x:[0-9]+}/{y:[0-9]+}.{ext}", tileAppHandler(requestTile)) return r @@ -432,7 +439,7 @@ func TileRouter() *mux.Router { func handleRequests() { // Get a configured router - r := TileRouter() + r := tileRouter() // Allow CORS from anywhere corsOrigins := viper.GetStringSlice("CORSOrigins") diff --git a/main_test.go b/main_test.go index 4fdad6e..a1da245 100644 --- a/main_test.go +++ b/main_test.go @@ -20,7 +20,7 @@ func TestMain(m *testing.M) { // viper.Set("DbConnection", os.Getenv("TEST_DATABASE_URL")) viper.Set("DbConnection", os.Getenv("dbname=ts")) - db, err := DbConnect() + db, err := dbConnect() if err != nil { os.Exit(1) } @@ -39,15 +39,15 @@ func TestDBNoTables(t *testing.T) { if !dbsetup { t.Skip("DB integration test suite setup failed, skipping") } - r := TileRouter() + r := tileRouter() request, _ := http.NewRequest("GET", "/index.json", nil) response := httptest.NewRecorder() r.ServeHTTP(response, request) assert.Equal(t, 200, response.Code, "OK response is expected") - json_result := strings.TrimSpace(response.Body.String()) - json_expect := "{}" - assert.Equal(t, json_expect, json_result, "empty json response is expected") + jsonResult := strings.TrimSpace(response.Body.String()) + jsonExpect := "{}" + assert.Equal(t, jsonExpect, jsonResult, "empty json response is expected") } // TestBasePath sets an alternate base path to check that handlers are @@ -63,7 +63,7 @@ func TestBasePath(t *testing.T) { for _, path := range paths { viper.Set("BasePath", path) - r := TileRouter() + r := tileRouter() request, _ := http.NewRequest("GET", "/test/index.json", nil) response := httptest.NewRecorder() r.ServeHTTP(response, request) diff --git a/tile.go b/tile.go index 3901d00..dc72c8d 100644 --- a/tile.go +++ b/tile.go @@ -38,7 +38,7 @@ func makeTile(vars map[string]string) (Tile, error) { return tile, e } -func (tile *Tile) Width() float64 { +func (tile *Tile) width() float64 { return math.Abs(tile.Bounds.Xmax - tile.Bounds.Xmin) } diff --git a/util.go b/util.go index 9675b57..655f8df 100644 --- a/util.go +++ b/util.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "strings" + "sync" "text/template" log "github.com/sirupsen/logrus" @@ -15,6 +16,10 @@ import ( "github.com/theckman/httpforwarded" ) +// cache SQL/HTML templates so repeated filesystem reads are not required +var globalTemplates map[string](*template.Template) = make(map[string](*template.Template)) +var globalTemplatesMutex = &sync.Mutex{} + // formatBaseURL takes a hostname (baseHost) and a base path // and joins them. Both are parsed as URLs (using net/url) and // then joined to ensure a properly formed URL. @@ -54,9 +59,9 @@ func serverURLBase(r *http.Request) string { // configuration option. func serverURLHost(r *http.Request) string { // Use configuration file settings if we have them - configUrl := viper.GetString("UrlBase") - if configUrl != "" { - return configUrl + configURL := viper.GetString("UrlBase") + if configURL != "" { + return configURL } // Preferred scheme @@ -95,9 +100,7 @@ func serverURLHost(r *http.Request) string { return fmt.Sprintf("%v://%v", ps, ph) } -var globalTemplates map[string](*template.Template) = make(map[string](*template.Template)) - -func getSqlTemplate(name string, tmpl string) *template.Template { +func getSQLTemplate(name string, tmpl string) *template.Template { tp, ok := globalTemplates[name] if ok { return tp @@ -107,13 +110,15 @@ func getSqlTemplate(name string, tmpl string) *template.Template { if err != nil { log.Fatal(err) } + globalTemplatesMutex.Lock() globalTemplates[name] = tp + globalTemplatesMutex.Unlock() return tp } -func renderSqlTemplate(name string, tmpl string, data interface{}) (string, error) { +func renderSQLTemplate(name string, tmpl string, data interface{}) (string, error) { var buf bytes.Buffer - t := getSqlTemplate(name, tmpl) + t := getSQLTemplate(name, tmpl) err := t.Execute(&buf, data) if err != nil { return string(buf.Bytes()), err