From 67576af64a6050e7138d24793fff3b7d64789c5c Mon Sep 17 00:00:00 2001 From: pcarana Date: Tue, 27 Oct 2020 18:22:11 -0600 Subject: [PATCH] Add argument '--init-tals' to fetch RIR TALs +Once utilized, FORT tries to download the TALs and exits. In order to download ARIN TAL, the user must explicitly accept its RPA by typing yes (ignoring case) at stdin. +Remove the write callback from HTTP download callers, it was unnecessary since every caller did the same thing. +Update the docs to include the new argument. --- docs/run.md | 4 +- docs/usage.md | 333 +++++++++++++++++++++++------------------ man/fort.8 | 101 ++++++++++++- src/Makefile.am | 2 + src/config.c | 65 ++++++++ src/config/init_tals.c | 246 ++++++++++++++++++++++++++++++ src/config/init_tals.h | 27 ++++ src/http/http.c | 101 +++++++++++-- src/http/http.h | 7 +- src/init.c | 111 ++++++++++++++ src/init.h | 8 + src/object/tal.c | 16 +- src/rrdp/rrdp_parser.c | 21 +-- test/http_test.c | 4 +- 14 files changed, 844 insertions(+), 202 deletions(-) create mode 100644 src/config/init_tals.c create mode 100644 src/config/init_tals.h create mode 100644 src/init.c create mode 100644 src/init.h diff --git a/docs/run.md b/docs/run.md index 1de4dab7..defbd55e 100644 --- a/docs/run.md +++ b/docs/run.md @@ -5,9 +5,11 @@ description: This is probably all you need, an RTR server will serve the ROAs re # {{ page.title }} -This is probably all you need, an RTR server will serve the ROAs resulting from a validation rooted at the trust anchors defined by the TALs contained at directory `--tal`: +This is probably all you need: fetch the RIR TALs and then start an RTR server that will serve the ROAs resulting from a validation rooted at the trust anchors defined by the TALs contained at directory `--tal`: {% highlight bash %} +fort --init-tals + fort \ --tal \ --local-repository \ diff --git a/docs/usage.md b/docs/usage.md index 83ad2c01..cfbaf0dc 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -13,48 +13,48 @@ description: Guide to use arguments of FORT Validator. 1. [`--help`](#--help) 2. [`--usage`](#--usage) 3. [`--version`](#--version) - 4. [`--tal`](#--tal) - 5. [`--local-repository`](#--local-repository) - 6. [`--work-offline`](#--work-offline) - 7. [`--shuffle-uris`](#--shuffle-uris) - 8. [`--maximum-certificate-depth`](#--maximum-certificate-depth) - 9. [`--mode`](#--mode) - 10. [`--server.address`](#--serveraddress) - 11. [`--server.port`](#--serverport) - 12. [`--server.backlog`](#--serverbacklog) - 13. [`--server.interval.validation`](#--serverintervalvalidation) - 14. [`--server.interval.refresh`](#--serverintervalrefresh) - 15. [`--server.interval.retry`](#--serverintervalretry) - 16. [`--server.interval.expire`](#--serverintervalexpire) - 17. [`--slurm`](#--slurm) - 18. [`--log.enabled`](#--logenabled) - 19. [`--log.level`](#--loglevel) - 20. [`--log.output`](#--logoutput) - 21. [`--log.color-output`](#--logcolor-output) - 22. [`--log.file-name-format`](#--logfile-name-format) - 23. [`--log.facility`](#--logfacility) - 24. [`--log.tag`](#--logtag) - 25. [`--validation-log.enabled`](#--validation-logenabled) - 26. [`--validation-log.level`](#--validation-loglevel) - 27. [`--validation-log.output`](#--validation-logoutput) - 28. [`--validation-log.color-output`](#--validation-logcolor-output) - 29. [`--validation-log.file-name-format`](#--validation-logfile-name-format) - 30. [`--validation-log.facility`](#--validation-logfacility) - 31. [`--validation-log.tag`](#--validation-logtag) - 32. [`--http.enabled`](#--httpenabled) - 33. [`--http.priority`](#--httppriority) - 34. [`--http.retry.count`](#--httpretrycount) - 35. [`--http.retry.interval`](#--httpretryinterval) - 36. [`--http.user-agent`](#--httpuser-agent) - 37. [`--http.connect-timeout`](#--httpconnect-timeout) - 38. [`--http.transfer-timeout`](#--httptransfer-timeout) - 39. [`--http.idle-timeout`](#--httpidle-timeout) - 40. [`--http.ca-path`](#--httpca-path) - 41. [`--output.roa`](#--outputroa) - 42. [`--output.bgpsec`](#--outputbgpsec) - 43. [`--asn1-decode-max-stack`](#--asn1-decode-max-stack) - 44. [`--stale-repository-period`](#--stale-repository-period) - 45. [`--configuration-file`](#--configuration-file) + 4. [`--init-tals`](#--init-tals) + 5. [`--tal`](#--tal) + 6. [`--local-repository`](#--local-repository) + 7. [`--work-offline`](#--work-offline) + 8. [`--shuffle-uris`](#--shuffle-uris) + 9. [`--maximum-certificate-depth`](#--maximum-certificate-depth) + 10. [`--mode`](#--mode) + 11. [`--server.address`](#--serveraddress) + 12. [`--server.port`](#--serverport) + 13. [`--server.backlog`](#--serverbacklog) + 14. [`--server.interval.validation`](#--serverintervalvalidation) + 15. [`--server.interval.refresh`](#--serverintervalrefresh) + 16. [`--server.interval.retry`](#--serverintervalretry) + 17. [`--server.interval.expire`](#--serverintervalexpire) + 18. [`--slurm`](#--slurm) + 19. [`--log.enabled`](#--logenabled) + 20. [`--log.level`](#--loglevel) + 21. [`--log.output`](#--logoutput) + 22. [`--log.color-output`](#--logcolor-output) + 23. [`--log.file-name-format`](#--logfile-name-format) + 24. [`--log.facility`](#--logfacility) + 25. [`--log.tag`](#--logtag) + 26. [`--validation-log.enabled`](#--validation-logenabled) + 27. [`--validation-log.level`](#--validation-loglevel) + 28. [`--validation-log.output`](#--validation-logoutput) + 29. [`--validation-log.color-output`](#--validation-logcolor-output) + 30. [`--validation-log.file-name-format`](#--validation-logfile-name-format) + 31. [`--validation-log.facility`](#--validation-logfacility) + 32. [`--validation-log.tag`](#--validation-logtag) + 33. [`--http.enabled`](#--httpenabled) + 34. [`--http.priority`](#--httppriority) + 35. [`--http.retry.count`](#--httpretrycount) + 36. [`--http.retry.interval`](#--httpretryinterval) + 37. [`--http.user-agent`](#--httpuser-agent) + 38. [`--http.connect-timeout`](#--httpconnect-timeout) + 39. [`--http.transfer-timeout`](#--httptransfer-timeout) + 40. [`--http.idle-timeout`](#--httpidle-timeout) + 41. [`--http.ca-path`](#--httpca-path) + 42. [`--output.roa`](#--outputroa) + 43. [`--output.bgpsec`](#--outputbgpsec) + 44. [`--asn1-decode-max-stack`](#--asn1-decode-max-stack) + 45. [`--stale-repository-period`](#--stale-repository-period) 46. [`--rsync.enabled`](#--rsyncenabled) 47. [`--rsync.priority`](#--rsyncpriority) 48. [`--rsync.strategy`](#--rsyncstrategy) @@ -63,10 +63,12 @@ description: Guide to use arguments of FORT Validator. 3. [`root-except-ta`](#root-except-ta) 49. [`--rsync.retry.count`](#--rsyncretrycount) 50. [`--rsync.retry.interval`](#--rsyncretryinterval) - 51. [`rsync.program`](#rsyncprogram) - 52. [`rsync.arguments-recursive`](#rsyncarguments-recursive) - 53. [`rsync.arguments-flat`](#rsyncarguments-flat) - 54. [`incidences`](#incidences) + 51. [`--configuration-file`](#--configuration-file) + 52. [`rsync.program`](#rsyncprogram) + 53. [`rsync.arguments-recursive`](#rsyncarguments-recursive) + 54. [`rsync.arguments-flat`](#rsyncarguments-flat) + 55. [`incidences`](#incidences) + 56. [`init-locations`](#init-locations) 3. [Deprecated arguments](#deprecated-arguments) 1. [`--sync-strategy`](#--sync-strategy) 2. [`--rrdp.enabled`](#--rrdpenabled) @@ -81,6 +83,7 @@ description: Guide to use arguments of FORT Validator. [--help] [--usage] [--version] + [--init-tals] [--configuration-file=] [--tal=|] [--local-repository=] @@ -204,9 +207,20 @@ $ {{ page.command }} --version fort {{ site.fort-latest-version }} {% endhighlight %} +### `--init-tals` + +- **Type:** String (Path to directory) +- **Availability:** `argv` only + +Download the RIR TALs into the existent local path directory and exit. + +This argument exists merely to have all TALs before running FORT validator, the directory path should be the same that will be set at the [`--tal`](#--tal) argument. + +By default, the 4 TALs that don't require a policy acceptance are downloaded from FORT validator's GitHub repository. ARIN TAL does require an explicit acceptance by the user, so it's downloaded only after the user accepts ARIN's RPA; this message is displayed at the terminal and only if the user accepts, ARIN TAL is also downloaded. + ### `--tal` -- **Type:** String (Path to file) +- **Type:** String (Path to file or directory) - **Availability:** `argv` and JSON Path to the _Trust Anchor Locator_ (TAL), or to a directory that contains TALs. @@ -741,6 +755,102 @@ Currently **all** the communication errors are logged at the validation log. Thi A value **equal to 0** means that the communication errors will be logged at once. +### `--rsync.enabled` + +- **Type:** Boolean (`true`, `false`) +- **Availability:** `argv` and JSON +- **Default:** `true` + +Enables RSYNC requests. + +If disabled (eg. `--rsync.enabled=false`), FORT validator won't download files nor directories via RSYNC, and will expect to find all repository files at [`--local-repository`](#--local-repository). + +### `--rsync.priority` + +- **Type:** Integer +- **Availability:** `argv` and JSON +- **Default:** 50 +- **Range:** 0--100 + +> ![img/warn.svg](img/warn.svg) By default, HTTPS requests are preferred over rsync requests. + +Assign priority to use RSYNC to fetch repository files. A higher value means a higher priority. + +This argument works along with [`--http.priority`](#--httppriority), since the higher value of the two arguments will result in the first protocol to utilize when fetching repositories files. Of course, this depends also on certificates information or the TAL URIs, since currently HTTP URIs are optional and not every RIR repository makes use of them. + +Whenever a certificate or a TAL has both RSYNC and HTTP URIs, the following criteria is followed to prioritize which one to use first: +- [`--rsync.priority`](#--rsyncpriority) **equals** [`--http.priority`](#--httppriority): use the order specified at the certificate or the TAL to fetch the corresponding URI. +- [`--rsync.priority`](#--rsyncpriority) **greater than** [`--http.priority`](#--httppriority): use RSYNC repository/TAL URI first; if there's an error fetching data, fallback to fetch HTTP repository/TAL URI. +- [`--rsync.priority`](#--rsyncpriority) **less than** [`--http.priority`](#--httppriority): use HTTP repository/TAL URI first; if there's an error fetching data, fallback to use RSYNC repository/TAL URI. + +### `--rsync.strategy` + +- **Type:** Enumeration (`strict`, `root`, `root-except-ta`) +- **Availability:** `argv` and JSON +- **Default:** `root-except-ta` + +rsync synchronization strategy. Commands the way rsync URLs are approached during downloads. + +#### `strict` + +> In order to enable this strategy, recompile using the flag: **_ENABLE\_STRICT\_STRATEGY_**. +> +> e.g. `$ make FORT_FLAGS='-DENABLE_STRICT_STRATEGY'` + +rsyncs every repository publication point separately. Only skips publication points that have already been downloaded during the current validation cycle. (Assuming each synchronization is recursive.) + +For example, suppose the validator gets certificates whose caRepository access methods (in their Subject Information Access extensions) point to the following publication points: + +1. `rsync://rpki.example.com/foo/bar/` +2. `rsync://rpki.example.com/foo/qux/` +3. `rsync://rpki.example.com/foo/bar/` +4. `rsync://rpki.example.com/foo/corge/grault/` +5. `rsync://rpki.example.com/foo/corge/` +6. `rsync://rpki.example.com/foo/corge/waldo/` + +A validator following the `strict` strategy would download `bar`, download `qux`, skip `bar`, download `corge/grault`, download `corge` and skip `corge/waldo`. + +Though this strategy is the only "strictly" correct one, it is also extremely slow. Its usage is _not_ recommended, unless your repository contains lots of spam files, awkward permissions or can't be found in a repository rooted in a URL that follows the regular expression "`rsync://.+/.+/`". + +#### `root` + +For each publication point found, guess the root of its repository and rsync that instead. Then skip +any subsequent children of said root. + +(To guess the root of a repository, the validator counts four slashes, and prunes the rest of the URL.) + +Reusing the caRepository URLs from the `strict` strategy (above) as example, a validator following the `root` strategy would download `rsync://rpki.example.com/foo`, and then skip everything else. + +Assuming that the repository is specifically structured to be found within as few roots as possible, and they contain minimal RPKI-unrelated noise files, this is the fastest synchronization strategy. At time of writing, this is true for all the current official repositories. + +#### `root-except-ta` + +Synchronizes the root certificate (the one pointed by the TAL) in `strict` mode, and once it's validated, synchronizes the rest of the repository in `root` mode. + +Useful if you want `root`, but the root certificate is separated from the rest of the repository. Also useful if you don't want the validator to download the entire repository without first confirming the integrity and legitimacy of the root certificate. + +### `--rsync.retry.count` + +- **Type:** Integer +- **Availability:** `argv` and JSON +- **Default:** 2 +- **Range:** 0--[`UINT_MAX`](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/limits.h.html) + +Maximum number of retries whenever there's an error executing an RSYNC. + +A value of **0** means **no retries**. + +Whenever is necessary to execute an RSYNC, the validator will try at least one time the execution. If there was an error executing the RSYNC, the validator will retry it at most `--rsync.retry.count` times, waiting [`--rsync.retry.interval`](#--rsyncretryinterval) seconds between each retry. + +### `--rsync.retry.interval` + +- **Type:** Integer +- **Availability:** `argv` and JSON +- **Default:** 5 +- **Range:** 0--[`UINT_MAX`](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/limits.h.html) + +Period of time (in seconds) to wait between each retry to execute an RSYNC. + ### `--configuration-file` - **Type:** String (Path to file) @@ -907,102 +1017,6 @@ $ {{ page.command }} \ $ # local-repository is "a", rsync.strategy is "strict" and maximum-certificate-depth is 8 {% endhighlight %} -### `--rsync.enabled` - -- **Type:** Boolean (`true`, `false`) -- **Availability:** `argv` and JSON -- **Default:** `true` - -Enables RSYNC requests. - -If disabled (eg. `--rsync.enabled=false`), FORT validator won't download files nor directories via RSYNC, and will expect to find all repository files at [`--local-repository`](#--local-repository). - -### `--rsync.priority` - -- **Type:** Integer -- **Availability:** `argv` and JSON -- **Default:** 50 -- **Range:** 0--100 - -> ![img/warn.svg](img/warn.svg) By default, HTTPS requests are preferred over rsync requests. - -Assign priority to use RSYNC to fetch repository files. A higher value means a higher priority. - -This argument works along with [`--http.priority`](#--httppriority), since the higher value of the two arguments will result in the first protocol to utilize when fetching repositories files. Of course, this depends also on certificates information or the TAL URIs, since currently HTTP URIs are optional and not every RIR repository makes use of them. - -Whenever a certificate or a TAL has both RSYNC and HTTP URIs, the following criteria is followed to prioritize which one to use first: -- [`--rsync.priority`](#--rsyncpriority) **equals** [`--http.priority`](#--httppriority): use the order specified at the certificate or the TAL to fetch the corresponding URI. -- [`--rsync.priority`](#--rsyncpriority) **greater than** [`--http.priority`](#--httppriority): use RSYNC repository/TAL URI first; if there's an error fetching data, fallback to fetch HTTP repository/TAL URI. -- [`--rsync.priority`](#--rsyncpriority) **less than** [`--http.priority`](#--httppriority): use HTTP repository/TAL URI first; if there's an error fetching data, fallback to use RSYNC repository/TAL URI. - -### `--rsync.strategy` - -- **Type:** Enumeration (`strict`, `root`, `root-except-ta`) -- **Availability:** `argv` and JSON -- **Default:** `root-except-ta` - -rsync synchronization strategy. Commands the way rsync URLs are approached during downloads. - -#### `strict` - -> In order to enable this strategy, recompile using the flag: **_ENABLE\_STRICT\_STRATEGY_**. -> -> e.g. `$ make FORT_FLAGS='-DENABLE_STRICT_STRATEGY'` - -rsyncs every repository publication point separately. Only skips publication points that have already been downloaded during the current validation cycle. (Assuming each synchronization is recursive.) - -For example, suppose the validator gets certificates whose caRepository access methods (in their Subject Information Access extensions) point to the following publication points: - -1. `rsync://rpki.example.com/foo/bar/` -2. `rsync://rpki.example.com/foo/qux/` -3. `rsync://rpki.example.com/foo/bar/` -4. `rsync://rpki.example.com/foo/corge/grault/` -5. `rsync://rpki.example.com/foo/corge/` -6. `rsync://rpki.example.com/foo/corge/waldo/` - -A validator following the `strict` strategy would download `bar`, download `qux`, skip `bar`, download `corge/grault`, download `corge` and skip `corge/waldo`. - -Though this strategy is the only "strictly" correct one, it is also extremely slow. Its usage is _not_ recommended, unless your repository contains lots of spam files, awkward permissions or can't be found in a repository rooted in a URL that follows the regular expression "`rsync://.+/.+/`". - -#### `root` - -For each publication point found, guess the root of its repository and rsync that instead. Then skip -any subsequent children of said root. - -(To guess the root of a repository, the validator counts four slashes, and prunes the rest of the URL.) - -Reusing the caRepository URLs from the `strict` strategy (above) as example, a validator following the `root` strategy would download `rsync://rpki.example.com/foo`, and then skip everything else. - -Assuming that the repository is specifically structured to be found within as few roots as possible, and they contain minimal RPKI-unrelated noise files, this is the fastest synchronization strategy. At time of writing, this is true for all the current official repositories. - -#### `root-except-ta` - -Synchronizes the root certificate (the one pointed by the TAL) in `strict` mode, and once it's validated, synchronizes the rest of the repository in `root` mode. - -Useful if you want `root`, but the root certificate is separated from the rest of the repository. Also useful if you don't want the validator to download the entire repository without first confirming the integrity and legitimacy of the root certificate. - -### `--rsync.retry.count` - -- **Type:** Integer -- **Availability:** `argv` and JSON -- **Default:** 2 -- **Range:** 0--[`UINT_MAX`](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/limits.h.html) - -Maximum number of retries whenever there's an error executing an RSYNC. - -A value of **0** means **no retries**. - -Whenever is necessary to execute an RSYNC, the validator will try at least one time the execution. If there was an error executing the RSYNC, the validator will retry it at most `--rsync.retry.count` times, waiting [`--rsync.retry.interval`](#--rsyncretryinterval) seconds between each retry. - -### `--rsync.retry.interval` - -- **Type:** Integer -- **Availability:** `argv` and JSON -- **Default:** 5 -- **Range:** 0--[`UINT_MAX`](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/limits.h.html) - -Period of time (in seconds) to wait between each retry to execute an RSYNC. - ### rsync.program - **Type:** String @@ -1033,11 +1047,42 @@ Fort will replace `"$REMOTE"` with the remote URL it needs to download, and `"$L ### `incidences` -- **Type:** JSON Object +- **Type:** JSON Object array - **Availability:** JSON only A listing of actions to be performed by validation upon encountering certain error conditions. See [Incidences](incidence.html). +### `init-locations` + +- **Type:** JSON Object array +- **Availability:** JSON only + +List of URLs from where the TALs will be fetched when [`--init-tals`](#--init-tals) is utilized. Each URL can have an optional `accept-message` that will be displayed at the terminal. When this message is displayed, the word **"yes"** is expected by FORT to download the corresponding TAL file; this way an explicit acceptance is obtained to comply with the printed message. + +By default it has 4 URLs from each TAL that doesn't require and explicit politics acceptance by the user, and 1 URL that does have an acceptance message so that FORT can proceed with its download. + +This is a JSON array of objects, where each object has a mandatory `url` member, and an optional `accept-message` member. The default value is: + +

+"init-locations": [
+	{
+		"url": "https://www.arin.net/resources/manage/rpki/arin.tal",
+		"accept-message": "Please download and read ARIN Relying Party Agreement (RPA) from https://www.arin.net/resources/manage/rpki/rpa.pdf. Once you've read it and if you agree ARIN RPA, type 'yes' to proceed with ARIN's TAL download:"
+	},
+	{
+		"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/lacnic.tal"
+	},
+	{
+		"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/ripe.tal"
+	},
+	{
+		"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/afrinic.tal"
+	},
+	{
+		"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/apnic.tal"
+	}
+]
+ ## Deprecated arguments ### `--sync-strategy` diff --git a/man/fort.8 b/man/fort.8 index 3a4f51a0..266b8697 100644 --- a/man/fort.8 +++ b/man/fort.8 @@ -5,7 +5,13 @@ fort \- RPKI certificate path validator and RTR server .SH SYNOPSIS .B fort -[\fIOPTIONS\fR] +[\fIOPTIONS\fR] +.P +.B fort +--init-tals=\fIPATH\fR +.P +.B fort +[-h|--help|--V|--version|--usage] .SH DESCRIPTION @@ -50,6 +56,22 @@ Print program version and exit. .RE .P +.B \-\-init-tals=\fIDIRECTORY\fR +.RS 4 +Download the RIR TALs into the existent local \fIDIRECTORY\fR and exit. +.P +This argument exists merely to have all TALs before running FORT validator, +the \fIDIRECTORY\fR should be the same that will be set at the \fI--tal\fR +argument. +.P +By default, the 4 TALs that don't require a policy acceptance are downloaded +from FORT validator's GitHub repository. ARIN TAL does require an explicit +acceptance by the user, so it's downloaded only after the user accepts ARIN's +RPA; this message is displayed at the terminal and only if the user accepts, +ARIN TAL is also downloaded. +.RE +.P + .BR \-f ", " \-\-configuration-file=\fIFILE\fR .RS 4 Path to a JSON file from where additional configuration will be read. @@ -180,6 +202,62 @@ match the actual file hash). [Default action: \fBerror\fR] .P More information about incidences can be consulted at FORT's web docs. .RE + +.P +.B init-locations +.RS 4 +List of URLs from where the TALs will be fetched when \fI--init-tals\fR is +utilized. Each URL can have an optional \fBaccept-message\fR that will be +displayed at the terminal. When this message is displayed, the word "yes" is +expected by FORT to download the corresponding TAL file; this way an explicit +acceptance is obtained to comply with the printed message. +.P +By default it has 4 URLs from each TAL that doesn't require and explicit +politics acceptance by the user, and 1 URL that does have an acceptance message +so that FORT can proceed with its download. +.P +This is a JSON array of objects, where each object has a mandatory \fBurl\fR +member, and an optional \fBaccept-message\fR member. The default value is: +.P +"init-locations": [ +.RS 2 +{ +.RS 2 +"url": "https://www.arin.net/resources/manage/rpki/arin.tal", +.br +"accept-message": "Please download and read ARIN Relying Party Agreement (RPA) +from https://www.arin.net/resources/manage/rpki/rpa.pdf. Once you've read it and +if you agree ARIN RPA, type 'yes' to proceed with ARIN's TAL download:" +.RE +}, +{ +.RS 2 +"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples\ +/tal/lacnic.tal" +.RE +}, +{ +.RS 2 +"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples\ +/tal/ripe.tal" +.RE +}, +{ +.RS 2 +"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples\ +/tal/afrinic.tal" +.RE +}, +{ +.RS 2 +"url": "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples\ +/tal/apnic.tal" +.RE +} +.RE +] +.RE + .RE .P @@ -1069,14 +1147,21 @@ By default, it has a value of \fI43200\fR (12 hours). .P .SH EXAMPLES -.B fort \-t /tmp/tal \-r /tmp/repository \-\-server.port 9323 +.B fort \-\-init-tals=/tmp/tal +.RS 4 +Download the 5 RIR TALs into the specified directory. ARIN TAL will be +downloaded only if the user accepts ARIN's RPA. +.RE +.P + +.B fort \-t /tmp/tal \-r /tmp/repository \-\-server.port=9323 .RS 4 Run FORT with all the default values, using a custom TALs directory, a custom repository directory as well, and binding the RTR server to port 9323. .RE .P -.B fort \-t /tmp/tal \-r /tmp/repository \-\-mode=standalone \-\-output.roa - +.B fort \-t /tmp/tal \-r /tmp/repository \-\-mode=standalone \-\-output.roa=- .RS 4 Run FORT as standalone and output ROAs CSV to the console. .RE @@ -1085,14 +1170,14 @@ Run FORT as standalone and output ROAs CSV to the console. .nf \fBfort \-t /tmp/tal \-r /tmp/repository \\ \-\-mode=standalone \\ - \-\-slurm /tmp/myslurm.slurm\fR + \-\-slurm=/tmp/myslurm.slurm\fR .fi .RS 4 Run FORT as standalone and using a SLURM file. .RE .P -.B fort --configuration-file conf.json +.B fort \-\-configuration-file=conf.json .RS 4 Run FORT using the JSON configuration file \fIconf.json\fR. .RE @@ -1100,9 +1185,9 @@ Run FORT using the JSON configuration file \fIconf.json\fR. .nf \fBfort \-t /tmp/tal \-r /tmp/repository \\ - \-\-server.address ::1 \-\-server.port 9323 \\ - \-\-server.interval.validation 1800 \\ - \-\-output.roa /tmp/roas.csv\fR + \-\-server.address=::1 \-\-server.port=9323 \\ + \-\-server.interval.validation=1800 \\ + \-\-output.roa=/tmp/roas.csv\fR .fi .RS 4 Run FORT with RTR server listening on IPv6 address \fI::1\fR, port 9323, diff --git a/src/Makefile.am b/src/Makefile.am index ccf10fc8..a6ab0da5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,6 +17,7 @@ fort_SOURCES += debug.h debug.c fort_SOURCES += delete_dir_daemon.h delete_dir_daemon.c fort_SOURCES += extension.h extension.c fort_SOURCES += file.h file.c +fort_SOURCES += init.h init.c fort_SOURCES += json_parser.c json_parser.h fort_SOURCES += line_file.h line_file.c fort_SOURCES += log.h log.c @@ -47,6 +48,7 @@ fort_SOURCES += config/filename_format.h config/filename_format.c fort_SOURCES += config/log_conf.h config/log_conf.c fort_SOURCES += config/mode.c config/mode.h fort_SOURCES += config/incidences.h config/incidences.c +fort_SOURCES += config/init_tals.h config/init_tals.c fort_SOURCES += config/rrdp_conf.h config/rrdp_conf.c fort_SOURCES += config/rsync_strategy.h config/rsync_strategy.c fort_SOURCES += config/str.c config/str.h diff --git a/src/config.c b/src/config.c index 2d03935e..4130035e 100644 --- a/src/config.c +++ b/src/config.c @@ -11,10 +11,12 @@ #include "common.h" #include "configure_ac.h" #include "file.h" +#include "init.h" #include "json_handler.h" #include "log.h" #include "config/boolean.h" #include "config/incidences.h" +#include "config/init_tals.h" #include "config/rrdp_conf.h" #include "config/str.h" #include "config/sync_strategy.h" @@ -193,6 +195,12 @@ struct rpki_config { /* Time period that must lapse to warn about a stale repository */ unsigned int stale_repository_period; + + /* Local dir where the TALs will be downloaded */ + char *init_tals; + + /* HTTPS URLS from where the TALS will be fetched */ + struct init_locations init_tal_locations; }; static void print_usage(FILE *, bool); @@ -718,6 +726,23 @@ static const struct option_field options[] = { .max = UINT_MAX, }, + { + .id = 11000, + .name = "init-tals", + .type = >_string, + .offset = offsetof(struct rpki_config, init_tals), + .doc = "Fetch the RIR's TAL files into the specified path", + .availability = AVAILABILITY_GETOPT, + }, + { + .id = 11001, + .name = "init-locations", + .type = >_init_tals_locations, + .offset = offsetof(struct rpki_config, init_tal_locations), + .doc = "Locations from where the TAL files will be downloaded, and its optional accept message", + .availability = AVAILABILITY_JSON, + }, + { 0 }, }; @@ -867,6 +892,18 @@ set_default_values(void) "$LOCAL", }; + static char const *init_locations_no_msg[] = { + "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/lacnic.tal", + "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/ripe.tal", + "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/afrinic.tal", + "https://raw.githubusercontent.com/NICMx/FORT-validator/master/examples/tal/apnic.tal", + }; + + static char const *init_locations_w_msg[] = { + "https://www.arin.net/resources/manage/rpki/arin.tal", + "Please download and read ARIN Relying Party Agreement (RPA) from https://www.arin.net/resources/manage/rpki/rpa.pdf. Once you've read it and if you agree ARIN RPA, type 'yes' to proceed with ARIN's TAL download:", + }; + int error; /* @@ -981,7 +1018,16 @@ set_default_values(void) rpki_config.asn1_decode_max_stack = 4096; /* 4kB */ rpki_config.stale_repository_period = 43200; /* 12 hours */ + rpki_config.init_tals = NULL; + error = init_locations_init(&rpki_config.init_tal_locations, + init_locations_no_msg, ARRAY_LEN(init_locations_no_msg), + init_locations_w_msg, ARRAY_LEN(init_locations_w_msg)); + if (error) + goto revert_init_locations; + return 0; +revert_init_locations: + free(rpki_config.validation_log.tag); revert_validation_log_tag: free(rpki_config.http.user_agent); revert_flat_array: @@ -1008,6 +1054,14 @@ valid_output_file(char const *path) static int validate_config(void) { + if (rpki_config.init_tals != NULL) { + if (!valid_file_or_dir(rpki_config.init_tals, false, true, + pr_op_errno)) + return pr_op_err("Invalid init TAL directory."); + /* Ignore the other checks */ + return 0; + } + if (rpki_config.tal == NULL) return pr_op_err("The TAL file/directory (--tal) is mandatory."); @@ -1132,6 +1186,17 @@ handle_flags_config(int argc, char **argv) error = validate_config(); + /* If present, nothing else is done */ + if (rpki_config.init_tals != NULL) { + if (error) + goto end; + error = init_tals_exec(&rpki_config.init_tal_locations, + rpki_config.init_tals); + free(long_opts); + free(short_opts); + exit(error); + } + log_start(); end: if (error) { diff --git a/src/config/init_tals.c b/src/config/init_tals.c new file mode 100644 index 00000000..07c114ab --- /dev/null +++ b/src/config/init_tals.c @@ -0,0 +1,246 @@ +#include "config/init_tals.h" + +#include +#include +#include +#include +#include "log.h" + +#define JSON_MEMBER_URL "url" +#define JSON_MEMBER_MESSAGE "accept-message" +// +//struct init_location { +// char *url; +// char *accept_message; +// SLIST_ENTRY(init_location) next; +//}; +// +//SLIST_HEAD(init_locations, init_location); + +static int +init_location_create(char const *url, struct init_location **result) +{ + struct init_location *tmp; + + tmp = malloc(sizeof(struct init_location)); + if (tmp == NULL) + return pr_enomem(); + + tmp->url = strdup(url); + if (tmp->url == NULL) { + free(tmp); + return pr_enomem(); + } + + tmp->accept_message = NULL; + + *result = tmp; + return 0; +} + +static void +init_location_destroy(struct init_location *location) +{ + if (location->accept_message != NULL) + free(location->accept_message); + free(location->url); + free(location); +} + +void +init_locations_cleanup(struct init_locations *locations) +{ + struct init_location *tmp; + + while (!SLIST_EMPTY(locations)) { + tmp = locations->slh_first; + SLIST_REMOVE_HEAD(locations, next); + init_location_destroy(tmp); + } +} + +void +__init_locations_cleanup(void *arg) +{ + init_locations_cleanup(arg); +} + +static int +init_locations_add_n_msg(struct init_locations *locations, char const *url) +{ + struct init_location *tmp; + int error; + + tmp = NULL; + error = init_location_create(url, &tmp); + if (error) + return error; + + SLIST_INSERT_HEAD(locations, tmp, next); + return 0; +} + +static int +init_locations_add_w_msg(struct init_locations *locations, char const *url, + char const *message) +{ + struct init_location *tmp; + int error; + + tmp = NULL; + error = init_location_create(url, &tmp); + if (error) + return error; + + tmp->accept_message = strdup(message); + if (tmp->accept_message == NULL) { + init_location_destroy(tmp); + return pr_enomem(); + } + + SLIST_INSERT_HEAD(locations, tmp, next); + return 0; +} + +int +init_locations_init(struct init_locations *locations, + char const *const *non_message, size_t non_message_len, + char const *const *with_message, size_t with_message_len) +{ + size_t i; + int error; + + SLIST_INIT(locations); + + for (i = 0; i < non_message_len; i++) { + error = init_locations_add_n_msg(locations, non_message[i]); + if (error) + goto cleanup; + } + + for (i = 0; i < with_message_len; i+=2) { + error = init_locations_add_w_msg(locations, with_message[i], + with_message[i + 1]); + if (error) + goto cleanup; + } + + return 0; +cleanup: + init_locations_cleanup(locations); + return error; +} + +int +init_locations_foreach(struct init_locations *locations, + init_locations_foreach_cb cb, void *arg) +{ + struct init_location *ptr; + int error; + + SLIST_FOREACH(ptr, locations, next) { + // FIXME TEST + pr_op_err("--> foreach = %s, '%s'", ptr->url, + (ptr->accept_message == NULL) ? "NULL" : ptr->accept_message); + + error = cb(ptr->url, ptr->accept_message, arg); + if (error) + return error; + } + + return 0; +} + +static int +parse_location(char const *name, size_t pos, json_t *json, char const **url, + char const **message) +{ + json_t *member; + + member = json_object_get(json, JSON_MEMBER_URL); + if (member == NULL) + return pr_op_err("'%s' array element #%zu requires the member '%s'.", + name, pos, JSON_MEMBER_URL); + + if (!json_is_string(member)) + return pr_op_err("'%s' array element #%zu '%s' member must be a string", + name, pos, JSON_MEMBER_URL); + + *url = json_string_value(member); + + /* Optional */ + member = json_object_get(json, JSON_MEMBER_MESSAGE); + if (member == NULL) { + *message = NULL; + return 0; + } + + if (!json_is_string(member)) + return pr_op_err("'%s' array element #%zu '%s' member must be a string", + name, pos, JSON_MEMBER_MESSAGE); + + *message = json_string_value(member); + + return 0; +} + +static int +init_tals_parse_json(struct option_field const *opt, json_t *json, void *result) +{ + struct init_locations *ptr; + json_t *elem; + size_t len; + size_t i; + char const *url; + char const *message; + int error; + + if (!json_is_array(json)) { + return pr_op_err("The '%s' element is not a JSON array", + opt->name); + } + + len = json_array_size(json); + + if (len == 0) { + /* Cleanup default value */ + init_locations_cleanup(result); + return 0; + } + + ptr = result; + + /* Remove the previous value (usually the default). */ + init_locations_cleanup(ptr); + + for (i = 0; i < len; i++) { + elem = json_array_get(json, i); + if (!json_is_object(elem)) + return pr_op_err("'%s' array element #%zu is not an object", + opt->name, i); + + url = NULL; + message = NULL; + error = parse_location(opt->name, i, elem, &url, &message); + if (error) + goto cleanup; + + if (message == NULL) + error = init_locations_add_n_msg(ptr, url); + else + error = init_locations_add_w_msg(ptr, url, message); + + if (error) + goto cleanup; + } + return 0; +cleanup: + init_locations_cleanup(ptr); + return error; +} + +const struct global_type gt_init_tals_locations = { + .print = NULL, + .parse.json = init_tals_parse_json, + .free = __init_locations_cleanup, +}; diff --git a/src/config/init_tals.h b/src/config/init_tals.h new file mode 100644 index 00000000..e19fe5c3 --- /dev/null +++ b/src/config/init_tals.h @@ -0,0 +1,27 @@ +#ifndef SRC_CONFIG_INIT_TALS_H_ +#define SRC_CONFIG_INIT_TALS_H_ + +#include +#include +#include "config/types.h" + +/* Struct where each URL and its optional message are stored */ +struct init_location { + char *url; + char *accept_message; + SLIST_ENTRY(init_location) next; +}; + +SLIST_HEAD(init_locations, init_location); + +extern const struct global_type gt_init_tals_locations; + +typedef int (*init_locations_foreach_cb)(char const *, char const *, void *); +int init_locations_foreach(struct init_locations *, init_locations_foreach_cb, + void *); + +int init_locations_init(struct init_locations *, char const *const *, size_t, + char const *const *, size_t); +void init_locations_cleanup(struct init_locations *); + +#endif /* SRC_CONFIG_INIT_TALS_H_ */ diff --git a/src/http/http.c b/src/http/http.c index e3ca9aac..7ecfab24 100644 --- a/src/http/http.c +++ b/src/http/http.c @@ -17,6 +17,8 @@ /* HTTP Response Code 400 (Bad Request) */ #define HTTP_BAD_REQUEST 400 +typedef size_t (http_write_cb)(unsigned char *, size_t, size_t, void *); + struct http_handler { CURL *curl; char errbuf[CURL_ERROR_SIZE]; @@ -164,9 +166,23 @@ http_easy_cleanup(struct http_handler *handler) curl_easy_cleanup(handler->curl); } +static size_t +write_cb(unsigned char *content, size_t size, size_t nmemb, void *arg) +{ + FILE *fd = arg; + size_t read = size * nmemb; + size_t written; + + written = fwrite(content, size, nmemb, fd); + if (written != nmemb) + return -EINVAL; + + return read; +} + static int -__http_download_file(struct rpki_uri *uri, http_write_cb cb, - long *response_code, long ims_value, long *cond_met, bool log_operation) +__http_download_file(struct rpki_uri *uri, long *response_code, long ims_value, + long *cond_met, bool log_operation) { char const *tmp_suffix = "_tmp"; struct http_handler handler; @@ -218,7 +234,7 @@ __http_download_file(struct rpki_uri *uri, http_write_cb cb, CURL_TIMECOND_IFMODSINCE); } error = http_fetch(&handler, uri_get_global(uri), response_code, - cond_met, log_operation, cb, out); + cond_met, log_operation, write_cb, out); if (error != EREQFAILED) break; @@ -264,23 +280,22 @@ release_tmp: /* * Try to download from global @uri into a local directory structure created - * from local @uri. The @cb should be utilized to write into a file; the file - * will be sent to @cb as the last argument (its a FILE reference). + * from local @uri. * * Return values: 0 on success, negative value on error, EREQFAILED if the * request to the server failed. */ int -http_download_file(struct rpki_uri *uri, http_write_cb cb, bool log_operation) +http_download_file(struct rpki_uri *uri, bool log_operation) { long response; long cond_met; - return __http_download_file(uri, cb, &response, 0, &cond_met, + return __http_download_file(uri, &response, 0, &cond_met, log_operation); } /* - * Fetch the file from @uri, write it using the @cb. + * Fetch the file from @uri. * * The HTTP request is made using the header 'If-Modified-Since' with a value * of @value (if @value is 0, the header isn't set). @@ -293,14 +308,14 @@ http_download_file(struct rpki_uri *uri, http_write_cb cb, bool log_operation) * < 0 an actual error happened. */ int -http_download_file_with_ims(struct rpki_uri *uri, http_write_cb cb, long value, +http_download_file_with_ims(struct rpki_uri *uri, long value, bool log_operation) { long response; long cond_met; int error; - error = __http_download_file(uri, cb, &response, value, &cond_met, + error = __http_download_file(uri, &response, value, &cond_met, log_operation); if (error) return error; @@ -320,7 +335,71 @@ http_download_file_with_ims(struct rpki_uri *uri, http_write_cb cb, long value, if (cond_met) return 0; - return __http_download_file(uri, cb, &response, 0, &cond_met, + return __http_download_file(uri, &response, 0, &cond_met, log_operation); } + +/* + * Downloads @remote to the absolute path @dest (no workspace nor directory + * structure is created). + */ +int +http_direct_download(char const *remote, char const *dest) +{ + char const *tmp_suffix = "_tmp"; + struct http_handler handler; + struct stat stat; + FILE *out; + long response_code; + long cond_met; + char *tmp_file, *tmp; + int error; + + tmp_file = strdup(dest); + if (tmp_file == NULL) + return pr_enomem(); + + tmp = realloc(tmp_file, strlen(tmp_file) + strlen(tmp_suffix) + 1); + if (tmp == NULL) { + error = pr_enomem(); + goto release_tmp; + } + + tmp_file = tmp; + strcat(tmp_file, tmp_suffix); + + error = file_write(tmp_file, &out, &stat); + if (error) + goto release_tmp; + + error = http_easy_init(&handler); + if (error) + goto close_file; + + response_code = 0; + cond_met = 0; + error = http_fetch(&handler, remote, &response_code, &cond_met, true, + write_cb, out); + http_easy_cleanup(&handler); + file_close(out); + if (error) + goto release_tmp; + + /* Overwrite the original file */ + error = rename(tmp_file, dest); + if (error) { + error = errno; + pr_val_errno(error, "Renaming temporal file from '%s' to '%s'", + tmp_file, dest); + goto release_tmp; + } + + free(tmp_file); + return 0; +close_file: + file_close(out); +release_tmp: + free(tmp_file); + return error; +} diff --git a/src/http/http.h b/src/http/http.h index 3ec2d305..717119d9 100644 --- a/src/http/http.h +++ b/src/http/http.h @@ -9,8 +9,9 @@ int http_init(void); void http_cleanup(void); -typedef size_t (http_write_cb)(unsigned char *, size_t, size_t, void *); -int http_download_file(struct rpki_uri *, http_write_cb, bool); -int http_download_file_with_ims(struct rpki_uri *, http_write_cb, long, bool); +int http_download_file(struct rpki_uri *, bool); +int http_download_file_with_ims(struct rpki_uri *, long, bool); + +int http_direct_download(char const *, char const *); #endif /* SRC_HTTP_HTTP_H_ */ diff --git a/src/init.c b/src/init.c new file mode 100644 index 00000000..ea98e1f3 --- /dev/null +++ b/src/init.c @@ -0,0 +1,111 @@ +#include "init.h" + +#include +#include +#include +#include + +#include "log.h" +#include "http/http.h" + +/* + * Quite simple: expect 'yes' from stdin (ignore case) + */ +static int +read_stdin(char const *file) +{ + char c; + + c = getchar(); + if (c != 'y' && c != 'Y') + goto err; + + c = getchar(); + if (c != 'e' && c != 'E') + goto err; + + c = getchar(); + if (c != 's' && c != 'S') + goto err; + + if (feof(stdin) || (c = getchar()) == '\n') + return 0; +err: + fprintf(stdout, + "\nWarning: The conditions weren't accepted, the TAL '%s' won't be downloaded.\n", + file); + return EINVAL; +} + +static int +fetch_url(char const *url, char const *accept_message, void *arg) +{ + char const *prefix = "https://"; + char const *dest_dir = arg; + char *dest_file; + char *dest; + size_t prefix_len; + size_t url_len; + size_t dest_dir_len; + size_t extra_slash; + size_t offset; + int error; + + prefix_len = strlen(prefix); + url_len = strlen(url); + dest_dir_len = strlen(dest_dir); + + if (url_len <= prefix_len || + strncasecmp(url, prefix, prefix_len) != 0) + return pr_op_err("Invalid HTTPS URL: '%s'", url); + + dest_file = strrchr(url, '/') + 1; + if (*dest_file == '\0') + return pr_op_err("HTTPS URL '%s' must be a file location", url); + + /* Each location must be an HTTPS URI */ + do { + if (accept_message == NULL) + break; + + fprintf(stdout, "%s\n", accept_message); + error = read_stdin(dest_file); + /* On error, let the other TALs to be downloaded */ + if (error) + return 0; + } while (0); + + extra_slash = (dest_dir[dest_dir_len - 1] == '/') ? 0 : 1; + + dest = malloc(dest_dir_len + extra_slash + strlen(dest_file) + 1); + if (dest == NULL) + return pr_enomem(); + + offset = 0; + strcpy(dest + offset, dest_dir); + offset += dest_dir_len; + if (extra_slash) { + strcpy(dest + offset, "/"); + offset += extra_slash; + } + strcpy(dest + offset, dest_file); + offset += strlen(dest_file); + dest[offset] = '\0'; + + error = http_direct_download(url, dest); + if (error) { + fprintf(stderr, "Couldn't fetch '%s'.\n", dest); + free(dest); + return error; + } + + fprintf(stdout, "Successfully fetched '%s'!\n", dest); + free(dest); + return 0; +} + +int +init_tals_exec(struct init_locations *source, char const *dest) +{ + return init_locations_foreach(source, fetch_url, (void *)dest); +} diff --git a/src/init.h b/src/init.h new file mode 100644 index 00000000..f37858ee --- /dev/null +++ b/src/init.h @@ -0,0 +1,8 @@ +#ifndef SRC_INIT_H_ +#define SRC_INIT_H_ + +#include "config/init_tals.h" + +int init_tals_exec(struct init_locations *, char const *); + +#endif /* SRC_INIT_H_ */ diff --git a/src/object/tal.c b/src/object/tal.c index f18ee470..d74c68cb 100644 --- a/src/object/tal.c +++ b/src/object/tal.c @@ -493,24 +493,10 @@ tal_get_spki(struct tal *tal, unsigned char const **buffer, size_t *len) *len = tal->spki_len; } -static size_t -write_http_cer(unsigned char *content, size_t size, size_t nmemb, void *arg) -{ - FILE *fd = arg; - size_t read = size * nmemb; - size_t written; - - written = fwrite(content, size, nmemb, fd); - if (written != nmemb) - return -EINVAL; - - return read; -} - static int handle_https_uri(struct rpki_uri *uri) { - return http_download_file(uri, write_http_cer, + return http_download_file(uri, reqs_errors_log_uri(uri_get_global(uri))); } diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c index b9593162..e1c4428c 100644 --- a/src/rrdp/rrdp_parser.c +++ b/src/rrdp/rrdp_parser.c @@ -95,31 +95,16 @@ rem_mft_from_list(struct visited_uris *visited_uris, char const *uri) return visited_uris_remove(visited_uris, uri); } -static size_t -write_local(unsigned char *content, size_t size, size_t nmemb, void *arg) -{ - FILE *fd = arg; - size_t read = size * nmemb; - size_t written; - - written = fwrite(content, size, nmemb, fd); - if (written != nmemb) - return -EINVAL; - - return read; -} - static int download_file(struct rpki_uri *uri, long last_update, bool log_operation) { int error; if (last_update > 0) - error = http_download_file_with_ims(uri, write_local, - last_update, log_operation); - else - error = http_download_file(uri, write_local, + error = http_download_file_with_ims(uri, last_update, log_operation); + else + error = http_download_file(uri, log_operation); /* * Since distinct files can be downloaded (notification, snapshot, diff --git a/test/http_test.c b/test/http_test.c index 5519c4db..affe9327 100644 --- a/test/http_test.c +++ b/test/http_test.c @@ -22,7 +22,7 @@ init_response(struct response *resp) } static size_t -write_cb(unsigned char *content, size_t size, size_t nmemb, void *arg) +write_buf_cb(unsigned char *content, size_t size, size_t nmemb, void *arg) { struct response *resp = arg; unsigned char *tmp; @@ -53,7 +53,7 @@ local_download(char const *url, long *response_code, struct response *resp) return error; error = http_fetch(&handler, url, response_code, &cond, false, - write_cb, resp); + write_buf_cb, resp); http_easy_cleanup(&handler); return error; } -- 2.47.2