zebra: Add kernel level graceful restart

<Initial Code from Praveen Chaudhary>

Add the a `--graceful_restart X` flag to zebra start that
now creates a timer that pops in X seconds and will go
through and remove all routes that are older than startup.

If graceful_restart is not specified then we will just pop
a timer that cleans everything up immediately.

Signed-off-by: Praveen Chaudhary <[email protected]>
Signed-off-by: Donald Sharp <[email protected]>

Author: donaldsharp

zebra/main.c

@@ -77,6 +77,8 @@ int allow_delete = 0;
 /* Don't delete kernel route. */
 int keep_kernel_mode = 0;
 
+int graceful_restart;
+
 bool v6_rr_semantics = false;
 
 #ifdef HAVE_NETLINK
@@ -95,6 +97,7 @@ struct option longopts[] = {
 	{"label_socket", no_argument, NULL, 'l'},
 	{"retain", no_argument, NULL, 'r'},
 	{"vrfdefaultname", required_argument, NULL, 'o'},
+	{"graceful_restart", required_argument, NULL, 'K'},
 #ifdef HAVE_NETLINK
 	{"vrfwnetns", no_argument, NULL, 'n'},
 	{"nl-bufsize", required_argument, NULL, 's'},
@@ -264,13 +267,14 @@ int main(int argc, char **argv)
 	char *netlink_fuzzing = NULL;
 #endif /* HANDLE_NETLINK_FUZZING */
 
+	graceful_restart = 0;
 	vrf_configure_backend(VRF_BACKEND_VRF_LITE);
 	logicalrouter_configure_backend(LOGICALROUTER_BACKEND_NETNS);
 
 	frr_preinit(&zebra_di, argc, argv);
 
 	frr_opt_add(
-		"bakz:e:l:o:r"
+		"bakz:e:l:o:rK:"
 #ifdef HAVE_NETLINK
 		"s:n"
 #endif
@@ -282,24 +286,25 @@ int main(int argc, char **argv)
 #endif /* HANDLE_NETLINK_FUZZING */
 		,
 		longopts,
-		"  -b, --batch           Runs in batch mode\n"
-		"  -a, --allow_delete    Allow other processes to delete zebra routes\n"
-		"  -z, --socket          Set path of zebra socket\n"
-		"  -e, --ecmp            Specify ECMP to use.\n"
-		"  -l, --label_socket    Socket to external label manager\n"
-		"  -k, --keep_kernel     Don't delete old routes which were installed by zebra.\n"
-		"  -r, --retain          When program terminates, retain added route by zebra.\n"
-		"  -o, --vrfdefaultname  Set default VRF name.\n"
+		"  -b, --batch              Runs in batch mode\n"
+		"  -a, --allow_delete       Allow other processes to delete zebra routes\n"
+		"  -z, --socket             Set path of zebra socket\n"
+		"  -e, --ecmp               Specify ECMP to use.\n"
+		"  -l, --label_socket       Socket to external label manager\n"
+		"  -k, --keep_kernel        Don't delete old routes which were installed by zebra.\n"
+		"  -r, --retain             When program terminates, retain added route by zebra.\n"
+		"  -o, --vrfdefaultname     Set default VRF name.\n"
+		"  -K, --graceful_restart  Graceful restart at the kernel level, timer in seconds for expiration\n"
 #ifdef HAVE_NETLINK
-		"  -n, --vrfwnetns       Use NetNS as VRF backend\n"
-		"  -s, --nl-bufsize      Set netlink receive buffer size\n"
-		"      --v6-rr-semantics Use v6 RR semantics\n"
+		"  -n, --vrfwnetns          Use NetNS as VRF backend\n"
+		"  -s, --nl-bufsize         Set netlink receive buffer size\n"
+		"      --v6-rr-semantics    Use v6 RR semantics\n"
 #endif /* HAVE_NETLINK */
 #if defined(HANDLE_ZAPI_FUZZING)
-		"  -c <file>             Bypass normal startup and use this file for testing of zapi\n"
+		"  -c <file>                Bypass normal startup and use this file for testing of zapi\n"
 #endif /* HANDLE_ZAPI_FUZZING */
 #if defined(HANDLE_NETLINK_FUZZING)
-		"  -w <file>             Bypass normal startup and use this file for testing of netlink input\n"
+		"  -w <file>                Bypass normal startup and use this file for testing of netlink input\n"
 #endif /* HANDLE_NETLINK_FUZZING */
 	);
 
@@ -319,6 +324,10 @@ int main(int argc, char **argv)
 			allow_delete = 1;
 			break;
 		case 'k':
+			if (graceful_restart) {
+				zlog_err("Graceful Restart initiated, we cannot keep the existing kernel routes");
+				return 1;
+			}
 			keep_kernel_mode = 1;
 			break;
 		case 'e':
@@ -350,6 +359,15 @@ int main(int argc, char **argv)
 		case 'r':
 			retain_mode = 1;
 			break;
+		case 'K':
+			if (keep_kernel_mode) {
+				zlog_err("Keep Kernel mode specified, graceful restart incompatible");
+				return 1;
+			}
+			graceful_restart = atoi(optarg);
+			if (graceful_restart > 5 * GRACEFUL_RESTART_TIME)
+				graceful_restart = 5 * GRACEFUL_RESTART_TIME;
+			break;
 #ifdef HAVE_NETLINK
 		case 's':
 			nl_rcvbufsize = atoi(optarg);
@@ -437,8 +455,9 @@ int main(int argc, char **argv)
 	*  will be equal to the current getpid(). To know about such routes,
 	* we have to have route_read() called before.
 	*/
-	if (!keep_kernel_mode)
-		rib_sweep_route();
+	zrouter.startup = time(NULL);
+	thread_add_timer(zrouter.master, rib_sweep_route,
+			 NULL, graceful_restart, NULL);
 
 	/* Needed for BSD routing socket. */
 	pid = getpid();

zebra/rib.h

@@ -351,7 +351,7 @@ extern struct route_entry *rib_lookup_ipv4(struct prefix_ipv4 *p,
 extern void rib_update(vrf_id_t vrf_id, rib_update_event_t event);
 extern void rib_update_table(struct route_table *table,
 			     rib_update_event_t event);
-extern void rib_sweep_route(void);
+extern int rib_sweep_route(struct thread *t);
 extern void rib_sweep_table(struct route_table *table);
 extern void rib_close_table(struct route_table *table);
 extern void rib_init(void);

zebra/zebra_rib.c

@@ -3139,12 +3139,15 @@ void rib_sweep_table(struct route_table *table)
 	struct route_entry *re;
 	struct route_entry *next;
 	struct nexthop *nexthop;
+	int64_t now = zrouter.startup;
 
 	if (!table)
 		return;
 
 	for (rn = route_top(table); rn; rn = srcdest_route_next(rn)) {
 		RNODE_FOREACH_RE_SAFE (rn, re, next) {
+			int64_t re_time = re->uptime;
+
 			if (IS_ZEBRA_DEBUG_RIB)
 				route_entry_dump(&rn->p, NULL, re);
 
@@ -3154,6 +3157,8 @@ void rib_sweep_table(struct route_table *table)
 			if (!CHECK_FLAG(re->flags, ZEBRA_FLAG_SELFROUTE))
 				continue;
 
+			if (now - re_time < 0)
+				continue;
 			/*
 			 * So we are starting up and have received
 			 * routes from the kernel that we have installed
@@ -3183,7 +3188,7 @@ void rib_sweep_table(struct route_table *table)
 }
 
 /* Sweep all RIB tables.  */
-void rib_sweep_route(void)
+int rib_sweep_route(struct thread *t)
 {
 	struct vrf *vrf;
 	struct zebra_vrf *zvrf;
@@ -3197,6 +3202,8 @@ void rib_sweep_route(void)
 	}
 
 	zebra_router_sweep_route();
+
+	return 0;
 }
 
 /* Remove specific by protocol routes from 'table'. */

zebra/zebra_router.h

@@ -110,8 +110,15 @@ struct zebra_router {
 	 * The EVPN instance, if any
 	 */
 	struct zebra_vrf *evpn_vrf;
+
+	/*
+	 * Time for when we sweep the rib from old routes
+	 */
+	time_t startup;
 };
 
+#define GRACEFUL_RESTART_TIME 60
+
 extern struct zebra_router zrouter;
 
 extern void zebra_router_init(void);