Initial import.

Code is working, but it just spits out to stdout.

Left to do is a refresh mechanism, and to serve it via http so something
like gortr can serve it to routers.

A sample configuration file is provided.

.gitignore

@@ -0,0 +1,3 @@
+ovplusplus
+*.json
+*.gz

cmd/merge.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"sync"
+
+	"github.com/dotwaffle/ovplusplus/pkg/irr"
+	"github.com/dotwaffle/ovplusplus/pkg/pfxops"
+	"github.com/dotwaffle/ovplusplus/pkg/rpki"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/sync/errgroup"
+)
+
+// mergeCmd implements the irr command.
+var mergeCmd = &cobra.Command{
+	Use:   "merge",
+	Short: "Create an export.json file based on IRR and RPKI data.",
+	Run: func(cmd *cobra.Command, args []string) {
+		ctx := context.TODO()
+		var mu sync.RWMutex
+		routes := make(map[string][]irr.Route)
+		e, eCtx := errgroup.WithContext(ctx)
+
+		// for each input, get the data
+		for _, src := range viper.GetStringSlice("file") {
+			src := src
+			e.Go(func() error {
+				srcRoutes, err := irr.FetchFile(eCtx, src)
+				if err != nil {
+					return err
+				}
+				log.Debug().Int("routes", len(srcRoutes)).Str("src", src).Msg("irrdb parsed")
+				mu.Lock()
+				routes[src] = srcRoutes
+				mu.Unlock()
+				return nil
+			})
+		}
+
+		for _, src := range viper.GetStringSlice("irrdb") {
+			src := src
+			e.Go(func() error {
+				srcRoutes, err := irr.FetchURL(eCtx, src)
+				if err != nil {
+					return err
+				}
+				log.Debug().Int("routes", len(srcRoutes)).Str("src", src).Msg("irrdb parsed")
+				mu.Lock()
+				routes[src] = srcRoutes
+				mu.Unlock()
+				return nil
+			})
+		}
+
+		if err := e.Wait(); err != nil {
+			log.Fatal().Err(err).Msg("irrdb read")
+		}
+
+		// produce some stats
+		merged := make(map[string][]string)
+		for _, r := range routes {
+			for _, rr := range r {
+				route := rr.Prefix.String()
+				merged[route] = append(merged[route], rr.Origin)
+			}
+		}
+		log.Debug().Int("routes", len(merged)).Msg("irrdb parsed total")
+		mergedStats := make(map[int]int)
+		for _, v := range merged {
+			mergedStats[len(v)]++
+		}
+		depth := make([]int, 0, len(mergedStats))
+		for k := range mergedStats {
+			depth = append(depth, k)
+		}
+		sort.Ints(depth)
+		for _, k := range depth {
+			log.Debug().Int("depth", k).Int("count", mergedStats[k]).Msg("irrdb depth stats")
+		}
+
+		// import RPKI ROA export
+		roas, err := rpki.Fetch(ctx, viper.GetString("rpki"))
+		if err != nil {
+			log.Fatal().Err(err).Msg("rpki.Fetch()")
+		}
+		log.Debug().Int("roas", len(roas)).Msg("rpki parsed")
+
+		// merge data
+		results, err := pfxops.Merge(roas, routes)
+		if err != nil {
+			log.Fatal().Err(err).Msg("pfxops.Merge()")
+		}
+		sort.SliceStable(results, func(i, j int) bool { return results[i].Prefix < results[j].Prefix })
+		log.Debug().Int("roas", len(results)).Msg("new total roas")
+
+		// dump the output to stdout
+		output, err := json.Marshal(rpki.Export{ROAs: results})
+		if err != nil {
+			log.Fatal().Err(err).Msg("json.Marshal()")
+		}
+		fmt.Println(string(output))
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(mergeCmd)
+
+	// fetch IRR data from a URL
+	mergeCmd.Flags().StringSliceP("irrdb", "i", []string{}, "url to fetch containing IRRDB data")
+	if err := viper.BindPFlag("irrdb", mergeCmd.Flags().Lookup("irrdb")); err != nil {
+		log.Fatal().Err(err).Msg("viper.BindPFlag(): irrdb")
+	}
+
+	// fetch IRR data from a local file
+	mergeCmd.Flags().StringSliceP("file", "f", []string{}, "local file containing IRRDB data")
+	if err := viper.BindPFlag("file", mergeCmd.Flags().Lookup("file")); err != nil {
+		log.Fatal().Err(err).Msg("viper.BindPFlag(): file")
+	}
+
+	// fetch RPKI data from a URL
+	mergeCmd.Flags().StringP("rpki", "r", "", "url to fetch containing RPKI ROA data")
+	if err := viper.BindPFlag("rpki", mergeCmd.Flags().Lookup("rpki")); err != nil {
+		log.Fatal().Err(err).Msg("viper.BindPFlag(): rpki")
+	}
+	// mergeCmd.MarkFlagRequired("rpki")
+
+	// use "orlonger" semantics instead of "exact" matching
+	mergeCmd.Flags().BoolP("longer", "l", false, "accept longer prefixes as well")
+	if err := viper.BindPFlag("longer", mergeCmd.Flags().Lookup("longer")); err != nil {
+		log.Fatal().Err(err).Msg("viper.BindPFlag(): longer")
+	}
+}

cmd/root.go

@@ -0,0 +1,111 @@
+// Package cmd provides the commands run by the ovplusplus command.
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	// name of the config file (without extension)
+	cfgName = "ovplusplus"
+	cfgFile string
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "ovplusplus",
+	Short: "Merge an IRRDB database with RPKI OV data into a single file.",
+	// TODO(dotwaffle): Long description, and usage
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	//	Run: func(cmd *cobra.Command, args []string) { },
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+		if viper.GetBool("debug") {
+			zerolog.SetGlobalLevel(zerolog.DebugLevel)
+		}
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		log.Fatal().Err(err).Msg("rootCmd.Execute()")
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "",
+		fmt.Sprintf("config file (default is $HOME/.config/%s/%s.yaml)", cfgName, cfgName))
+
+	rootCmd.PersistentFlags().BoolP("debug", "d", false, "output debug logging messages")
+	if err := viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug")); err != nil {
+		log.Fatal().Err(err).Msg("viper.BindPFlag(): debug")
+	}
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	if cfgFile != "" {
+		// Use config file from the flag.
+		viper.SetConfigFile(cfgFile)
+	} else {
+		viper.SetConfigName(cfgName)
+
+		// Search config in local directory.
+		viper.AddConfigPath(".")
+
+		// Search config in home directory (preferably in XDG config).
+		cfgDir, err := os.UserConfigDir()
+		if err != nil {
+			log.Fatal().Err(err).Msg("os.UserConfigDir()")
+		}
+		viper.AddConfigPath(filepath.Join(cfgDir, cfgName)) // subdir
+		viper.AddConfigPath(cfgDir)
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			log.Fatal().Err(err).Msg("os.UserHomeDir()")
+		}
+		viper.AddConfigPath(homeDir)
+
+		// Search config in other places.
+		viper.AddConfigPath(filepath.Join("/etc", cfgName))
+		viper.AddConfigPath("/etc")
+	}
+
+	viper.AutomaticEnv() // read in environment variables that match
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err == nil {
+		log.Info().Str("file", viper.ConfigFileUsed()).Msg("using stored config")
+	}
+}
+
+// writeConfigCmd writes the currently set configuration out.
+var writeConfigCmd = &cobra.Command{
+	Use:   "write-config",
+	Short: "Writes the currently set configuration out.",
+	Args:  cobra.MaximumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		switch len(args) {
+		case 0:
+			viper.WriteConfigAs(fmt.Sprintf("./%s.yaml", cfgName))
+		case 1:
+			viper.WriteConfigAs(args[0])
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(writeConfigCmd)
+}

go.mod

@@ -0,0 +1,12 @@
+module github.com/dotwaffle/ovplusplus
+
+go 1.14
+
+require (
+	github.com/jlaffaye/ftp v0.0.0-20200331144919-d4caf6ffcab8
+	github.com/rs/zerolog v1.18.0
+	github.com/spf13/cobra v0.0.7
+	github.com/spf13/viper v1.6.2
+	github.com/yl2chen/cidranger v1.0.0
+	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+)