Recover native tunnel failures safely

app/proguard-rules.pro

@@ -35,7 +35,7 @@
 -keep class eu.faircode.netguard.ResourceRecord { *; }
 -keep class eu.faircode.netguard.Usage { *; }
 -keep class eu.faircode.netguard.ServiceSinkhole {
-    void nativeExit(java.lang.String);
+    void nativeExit(int, java.lang.String);
     void nativeError(int, java.lang.String);
     void logPacket(eu.faircode.netguard.Packet);
     void dnsResolved(eu.faircode.netguard.ResourceRecord);

app/src/main/java/eu/faircode/netguard/NativeFailureRecoveryPolicy.java

@@ -0,0 +1,58 @@
+/*
+ * This file is part of TrackerControl.
+ *
+ * TrackerControl is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ */
+
+package eu.faircode.netguard;
+
+final class NativeFailureRecoveryPolicy {
+    static final long NO_RETRY = -1L;
+    static final int FILE_DESCRIPTOR_LIMIT_ERROR = 24;
+
+    private final int maxRetries;
+    private final long initialDelayMs;
+    private final long stableWindowMs;
+
+    private int failures;
+    private long lastFailureMs = Long.MIN_VALUE;
+
+    NativeFailureRecoveryPolicy(int maxRetries, long initialDelayMs, long stableWindowMs) {
+        if (maxRetries < 1 || initialDelayMs < 0 || stableWindowMs < 0)
+            throw new IllegalArgumentException();
+
+        this.maxRetries = maxRetries;
+        this.initialDelayMs = initialDelayMs;
+        this.stableWindowMs = stableWindowMs;
+    }
+
+    synchronized long onFailure(long nowMs) {
+        if (lastFailureMs != Long.MIN_VALUE &&
+                (nowMs < lastFailureMs || nowMs - lastFailureMs >= stableWindowMs))
+            failures = 0;
+
+        lastFailureMs = nowMs;
+        if (failures >= maxRetries)
+            return NO_RETRY;
+
+        // Exponential backoff, clamped to avoid signed-long overflow when
+        // maxRetries (and thus the shift amount) is large.
+        long delay = (failures < Long.SIZE - 1 && initialDelayMs <= (Long.MAX_VALUE >> failures))
+                ? initialDelayMs << failures
+                : Long.MAX_VALUE;
+        failures++;
+        return delay;
+    }
+
+    synchronized void reset() {
+        failures = 0;
+        lastFailureMs = Long.MIN_VALUE;
+    }
+
+    static boolean isFileDescriptorExhaustion(int error) {
+        return error == FILE_DESCRIPTOR_LIMIT_ERROR;
+    }
+}

app/src/main/java/eu/faircode/netguard/ServiceSinkhole.java

@@ -175,9 +175,9 @@ public class ServiceSinkhole extends VpnService {
 
     private static Object jni_lock = new Object();
     private static long jni_context = 0;
-    private Thread tunnelThread = null;
+    private volatile Thread tunnelThread = null;
     private ServiceSinkhole.Builder last_builder = null;
-    private ParcelFileDescriptor vpn = null;
+    private volatile ParcelFileDescriptor vpn = null;
     private boolean temporarilyStopped = false;
 
     private static long last_hosts_modified = 0;
@@ -196,6 +196,25 @@ public class ServiceSinkhole extends VpnService {
     private volatile LogHandler logHandler;
     private volatile StatsHandler statsHandler;
 
+    private static final int NATIVE_RECOVERY_MAX_RETRIES = 5;
+    private static final long NATIVE_RECOVERY_INITIAL_DELAY_MS = 1_000L;
+    private static final long NATIVE_RECOVERY_STABLE_WINDOW_MS = 2 * 60_000L;
+    private final NativeFailureRecoveryPolicy nativeRecoveryPolicy = new NativeFailureRecoveryPolicy(
+            NATIVE_RECOVERY_MAX_RETRIES,
+            NATIVE_RECOVERY_INITIAL_DELAY_MS,
+            NATIVE_RECOVERY_STABLE_WINDOW_MS);
+    private final Handler nativeRecoveryHandler = new Handler(Looper.getMainLooper());
+    private final Runnable nativeRecoveryRunnable = () -> {
+        SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(ServiceSinkhole.this);
+        if (!prefs.getBoolean("enabled", false) || temporarilyStopped || vpn == null) {
+            Log.i(TAG, "Skipping native recovery because protection is no longer active");
+            return;
+        }
+
+        Log.i(TAG, "Retrying native tunnel after failure");
+        ServiceSinkhole.reload("native tunnel recovery", ServiceSinkhole.this, false);
+    };
+
     // Cached preferences for shouldTrackApp() - refreshed in reload()
     private volatile SharedPreferences cachedTrackerProtectPrefs;
     private volatile boolean cachedManageSystem;
@@ -478,6 +497,8 @@ public void onCallStateChanged(int state, String incomingNumber) {
                         break;
 
                     case start:
+                        if (vpn == null)
+                            cancelNativeRecovery(true);
                         start();
                         break;
 
@@ -486,6 +507,7 @@ public void onCallStateChanged(int state, String incomingNumber) {
                         break;
 
                     case stop:
+                        cancelNativeRecovery(true);
                         stop(temporarilyStopped);
                         break;
 
@@ -1784,6 +1806,7 @@ public void run() {
             Log.i(TAG, "Started tunnel thread");
         }
 
+        cancelNativeRecovery(false);
         return true;
     }
 
@@ -2088,21 +2111,48 @@ private void stopVPN(ParcelFileDescriptor pfd) {
     }
 
     // Called from native code
-    private void nativeExit(String reason) {
-        Log.w(TAG, "Native exit reason=" + reason);
+    private void nativeExit(int error, String reason) {
+        Log.w(TAG, "Native exit error=" + error + " reason=" + reason);
         if (reason != null) {
-            showErrorNotification(reason);
-
             SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
-            prefs.edit().putBoolean("enabled", false).apply();
-            WidgetMain.updateWidgets(this);
+
+            // A native exit while TC is disabled or intentionally paused is expected: stay silent.
+            if (!prefs.getBoolean("enabled", false) || temporarilyStopped)
+                return;
+
+            long delayMs = nativeRecoveryPolicy.onFailure(SystemClock.elapsedRealtime());
+            if (delayMs == NativeFailureRecoveryPolicy.NO_RETRY) {
+                // Recovery budget exhausted: TC is going down, so surface the failure to the user.
+                Log.e(TAG, "Native recovery retry budget exhausted");
+                showErrorNotification(NativeFailureRecoveryPolicy.isFileDescriptorExhaustion(error)
+                        ? getString(R.string.msg_native_fd_recovery_exhausted)
+                        : reason);
+                cancelNativeRecovery(false);
+                prefs.edit().putBoolean("enabled", false).apply();
+                WidgetMain.updateWidgets(this);
+                ServiceSinkhole.stop("native recovery exhausted", this, false);
+                return;
+            }
+
+            // Transient failure: recover transparently, without alarming the user.
+            nativeRecoveryHandler.removeCallbacks(nativeRecoveryRunnable);
+            nativeRecoveryHandler.postDelayed(nativeRecoveryRunnable, delayMs);
+            Log.w(TAG, "Scheduled native recovery in " + delayMs + " ms");
         }
     }
 
     // Called from native code
     private void nativeError(int error, String message) {
         Log.w(TAG, "Native error " + error + ": " + message);
-        showErrorNotification(message);
+        showErrorNotification(NativeFailureRecoveryPolicy.isFileDescriptorExhaustion(error)
+                ? getString(R.string.msg_native_fd_error)
+                : message);
+    }
+
+    private void cancelNativeRecovery(boolean resetBudget) {
+        nativeRecoveryHandler.removeCallbacks(nativeRecoveryRunnable);
+        if (resetBudget)
+            nativeRecoveryPolicy.reset();
     }
 
     // Called from native code
@@ -3260,6 +3310,7 @@ public void onDestroy() {
 
             // Cancel any debounced network-change reload so it doesn't fire post-teardown
             networkReloadDebounceHandler.removeCallbacksAndMessages(NETWORK_RELOAD_TOKEN);
+            cancelNativeRecovery(true);
 
             commandLooper.quit();
             logLooper.quit();

app/src/main/jni/netguard/ip.c

@@ -74,12 +74,13 @@ int check_tun(const struct arguments *args,
     if (ev->events & EPOLLERR) {
         log_android(ANDROID_LOG_ERROR, "tun %d exception", args->tun);
         if (fcntl(args->tun, F_GETFL) < 0) {
+            int error = errno;
             log_android(ANDROID_LOG_ERROR, "fcntl tun %d F_GETFL error %d: %s",
-                        args->tun, errno, strerror(errno));
-            report_exit(args, "fcntl tun %d F_GETFL error %d: %s",
-                        args->tun, errno, strerror(errno));
+                        args->tun, error, strerror(error));
+            report_exit(args, error, "fcntl tun %d F_GETFL error %d: %s",
+                        args->tun, error, strerror(error));
         } else
-            report_exit(args, "tun %d exception", args->tun);
+            report_exit(args, 0, "tun %d exception", args->tun);
         return -1;
     }
 
@@ -88,16 +89,17 @@ int check_tun(const struct arguments *args,
         uint8_t *buffer = ng_malloc(get_mtu(), "tun read");
         ssize_t length = read(args->tun, buffer, get_mtu());
         if (length < 0) {
+            int error = errno;
             ng_free(buffer, __FILE__, __LINE__);
 
             log_android(ANDROID_LOG_ERROR, "tun %d read error %d: %s",
-                        args->tun, errno, strerror(errno));
-            if (errno == EINTR || errno == EAGAIN)
+                        args->tun, error, strerror(error));
+            if (error == EINTR || error == EAGAIN)
                 // Retry later
                 return 0;
             else {
-                report_exit(args, "tun %d read error %d: %s",
-                            args->tun, errno, strerror(errno));
+                report_exit(args, error, "tun %d read error %d: %s",
+                            args->tun, error, strerror(error));
                 return -1;
             }
         } else if (length > 0) {
@@ -119,7 +121,7 @@ int check_tun(const struct arguments *args,
             ng_free(buffer, __FILE__, __LINE__);
 
             log_android(ANDROID_LOG_ERROR, "tun %d empty read", args->tun);
-            report_exit(args, "tun %d empty read", args->tun);
+            report_exit(args, 0, "tun %d empty read", args->tun);
             return -1;
         }
     }

app/src/main/jni/netguard/netguard.c

@@ -476,10 +476,10 @@ Java_eu_faircode_netguard_Util_is_1numeric_1address(JNIEnv *env, jclass type, js
     return numeric;
 }
 
-void report_exit(const struct arguments *args, const char *fmt, ...) {
+void report_exit(const struct arguments *args, int error, const char *fmt, ...) {
     jclass cls = (*args->env)->GetObjectClass(args->env, args->instance);
     ng_add_alloc(cls, "cls");
-    jmethodID mid = jniGetMethodID(args->env, cls, "nativeExit", "(Ljava/lang/String;)V");
+    jmethodID mid = jniGetMethodID(args->env, cls, "nativeExit", "(ILjava/lang/String;)V");
 
     jstring jreason = NULL;
     if (fmt != NULL) {
@@ -492,7 +492,7 @@ void report_exit(const struct arguments *args, const char *fmt, ...) {
         va_end(argptr);
     }
 
-    (*args->env)->CallVoidMethod(args->env, args->instance, mid, jreason);
+    (*args->env)->CallVoidMethod(args->env, args->instance, mid, error, jreason);
     jniCheckException(args->env);
 
     if (jreason != NULL) {

app/src/main/jni/netguard/netguard.h

@@ -354,7 +354,7 @@ void handle_signal(int sig, siginfo_t *info, void *context);
 
 void *handle_events(void *a);
 
-void report_exit(const struct arguments *args, const char *fmt, ...);
+void report_exit(const struct arguments *args, int error, const char *fmt, ...);
 
 void report_error(const struct arguments *args, jint error, const char *fmt, ...);
 

app/src/main/jni/netguard/session.c

@@ -57,9 +57,11 @@ void *handle_events(void *a) {
     // Open epoll file
     int epoll_fd = epoll_create(1);
     if (epoll_fd < 0) {
-        log_android(ANDROID_LOG_ERROR, "epoll create error %d: %s", errno, strerror(errno));
-        report_exit(args, "epoll create error %d: %s", errno, strerror(errno));
+        int error = errno;
+        log_android(ANDROID_LOG_ERROR, "epoll create error %d: %s", error, strerror(error));
+        report_exit(args, error, "epoll create error %d: %s", error, strerror(error));
         args->ctx->stopping = 1;
+        goto cleanup;
     }
 
     // Monitor stop events
@@ -68,9 +70,11 @@ void *handle_events(void *a) {
     ev_pipe.events = EPOLLIN | EPOLLERR;
     ev_pipe.data.ptr = &ev_pipe;
     if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, args->ctx->pipefds[0], &ev_pipe)) {
-        log_android(ANDROID_LOG_ERROR, "epoll add pipe error %d: %s", errno, strerror(errno));
-        report_exit(args, "epoll add pipe error %d: %s", errno, strerror(errno));
+        int error = errno;
+        log_android(ANDROID_LOG_ERROR, "epoll add pipe error %d: %s", error, strerror(error));
+        report_exit(args, error, "epoll add pipe error %d: %s", error, strerror(error));
         args->ctx->stopping = 1;
+        goto cleanup;
     }
 
     // Monitor tun events
@@ -79,9 +83,11 @@ void *handle_events(void *a) {
     ev_tun.events = EPOLLIN | EPOLLERR;
     ev_tun.data.ptr = NULL;
     if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, args->tun, &ev_tun)) {
-        log_android(ANDROID_LOG_ERROR, "epoll add tun error %d: %s", errno, strerror(errno));
-        report_exit(args, "epoll add tun error %d: %s", errno, strerror(errno));
+        int error = errno;
+        log_android(ANDROID_LOG_ERROR, "epoll add tun error %d: %s", error, strerror(error));
+        report_exit(args, error, "epoll add tun error %d: %s", error, strerror(error));
         args->ctx->stopping = 1;
+        goto cleanup;
     }
 
     // Loop
@@ -181,15 +187,16 @@ void *handle_events(void *a) {
                                recheck ? EPOLL_MIN_CHECK : timeout * 1000);
 
         if (ready < 0) {
-            if (errno == EINTR) {
+            int error = errno;
+            if (error == EINTR) {
                 log_android(ANDROID_LOG_DEBUG, "epoll interrupted tun %d", args->tun);
                 continue;
             } else {
                 log_android(ANDROID_LOG_ERROR,
                             "epoll tun %d error %d: %s",
-                            args->tun, errno, strerror(errno));
-                report_exit(args, "epoll tun %d error %d: %s",
-                            args->tun, errno, strerror(errno));
+                            args->tun, error, strerror(error));
+                report_exit(args, error, "epoll tun %d error %d: %s",
+                            args->tun, error, strerror(error));
                 break;
             }
         }
@@ -270,6 +277,7 @@ void *handle_events(void *a) {
         }
     }
 
+cleanup:
     // Close epoll file
     if (epoll_fd >= 0 && close(epoll_fd))
         log_android(ANDROID_LOG_ERROR,
@@ -368,4 +376,3 @@ void check_allowed(const struct arguments *args) {
         s = s->next;
     }
 }
-

app/src/main/res/values/strings.xml

@@ -316,6 +316,8 @@ For this reason, please allow a VPN connection in the next dialog.
 Your internet traffic is not being sent to a remote VPN server.</string>
     <string name="msg_autostart">TrackerControl could not start automatically. This is likely because of a bug in your Android version.</string>
     <string name="msg_error">An unexpected error has occurred: \'%s\'</string>
+    <string name="msg_native_fd_recovery_exhausted">The VPN repeatedly ran out of file descriptors. Protection has stopped after all automatic restart attempts failed.</string>
+    <string name="msg_native_fd_error">The VPN ran out of file descriptors. Close unused apps or restart your device, then restart protection.</string>
     <string name="msg_doh_unreachable">Secure DNS (DoH) cannot be reached even though you have an active internet connection. If this happens repeatedly, consider disabling it in the Network settings.</string>
     <string name="msg_start_failed">Android refused to start the VPN service at this moment. This is likely because of a bug in your Android version.</string>
     <string name="msg_try">Try TrackerControl</string>