Merge pull request obsproject#243 from amazon-contributing/ruwen/fix-…

…encoder-group-destroy

libobs: Fix destroy_on_stop functionality of encoder groups

libobs/obs-encoder.c

@@ -393,13 +393,8 @@ static void remove_connection(struct obs_encoder *encoder, bool shutdown)
 			group->encoders_reconfigured = 0;
 			group->encoders_updated_next_pts = 0;
 			group->reconfigure_again = false;
-
-			if (group->destroy_on_stop)
-				obs_encoder_group_actually_destroy(group);
 		}
-
-		if (encoder->encoder_group)
-			pthread_mutex_unlock(&encoder->encoder_group->mutex);
+		pthread_mutex_unlock(&group->mutex);
 	}
 
 	/* obs_encoder_shutdown locks init_mutex, so don't call it on encode
@@ -409,6 +404,8 @@ static void remove_connection(struct obs_encoder *encoder, bool shutdown)
 	 * up again */
 	if (shutdown)
 		obs_encoder_shutdown(encoder);
+	encoder->initialized = false;
+
 	set_encoder_active(encoder, false);
 }
 
@@ -859,14 +856,20 @@ void obs_encoder_start(obs_encoder_t *encoder,
 	pthread_mutex_unlock(&encoder->init_mutex);
 }
 
-static inline bool obs_encoder_stop_internal(
-	obs_encoder_t *encoder,
-	void (*new_packet)(void *param, struct encoder_packet *packet),
-	void *param)
+void obs_encoder_stop(obs_encoder_t *encoder,
+		      void (*new_packet)(void *param,
+					 struct encoder_packet *packet),
+		      void *param)
 {
 	bool last = false;
 	size_t idx;
 
+	if (!obs_encoder_valid(encoder, "obs_encoder_stop"))
+		return;
+	if (!obs_ptr_valid(new_packet, "obs_encoder_stop"))
+		return;
+
+	pthread_mutex_lock(&encoder->init_mutex);
 	pthread_mutex_lock(&encoder->callbacks_mutex);
 
 	idx = get_callback_idx(encoder, new_packet, param);
@@ -883,34 +886,33 @@ static inline bool obs_encoder_stop_internal(
 
 	if (last) {
 		remove_connection(encoder, true);
-		encoder->initialized = false;
+		pthread_mutex_unlock(&encoder->init_mutex);
+
+		struct obs_encoder_group *group = encoder->encoder_group;
 
-		if (encoder->destroy_on_stop) {
-			pthread_mutex_unlock(&encoder->init_mutex);
+		if (encoder->destroy_on_stop)
 			obs_encoder_actually_destroy(encoder);
-			return true;
-		}
-	}
 
-	return false;
-}
+		if (!group)
+			return;
 
-void obs_encoder_stop(obs_encoder_t *encoder,
-		      void (*new_packet)(void *param,
-					 struct encoder_packet *packet),
-		      void *param)
-{
-	bool destroyed;
+		/* destroying the group all the way back here prevents a race where destruction
+		 * of the group can prematurely destroy the encoder within internal functions.
+		 * this is the point where it is safe to destroy the group, even if the encoder
+		 * is then also destroyed */
+		pthread_mutex_lock(&group->mutex);
+		if (group->destroy_on_stop &&
+		    group->num_encoders_started == 0) {
+			obs_encoder_group_actually_destroy(group);
+		} else {
+			pthread_mutex_unlock(&group->mutex);
+		}
 
-	if (!obs_encoder_valid(encoder, "obs_encoder_stop"))
-		return;
-	if (!obs_ptr_valid(new_packet, "obs_encoder_stop"))
+		/* init_mutex already unlocked */
 		return;
+	}
 
-	pthread_mutex_lock(&encoder->init_mutex);
-	destroyed = obs_encoder_stop_internal(encoder, new_packet, param);
-	if (!destroyed)
-		pthread_mutex_unlock(&encoder->init_mutex);
+	pthread_mutex_unlock(&encoder->init_mutex);
 }
 
 const char *obs_encoder_get_codec(const obs_encoder_t *encoder)
@@ -1463,7 +1465,6 @@ void full_stop(struct obs_encoder *encoder)
 		pthread_mutex_unlock(&encoder->callbacks_mutex);
 
 		remove_connection(encoder, false);
-		encoder->initialized = false;
 	}
 }