Fix the order of HSH_Fail() and ObjSetState()

The previous patch 4130055c went a bit too
far in it's mission to reorder events, and included putting HSH_Fail()
after ObjSetState(). That caused problems for code looking at the
OC_F_FAILED to learn about failed fetches.

Change the order of this back to normal, and move the call to HSH_Fail()
into ObjSetState(), so that the order can be shown and the caveats
properly commented.

bin/varnishd/cache/cache_fetch.c

@@ -809,7 +809,6 @@ vbf_stp_fail(struct worker *wrk, const struct busyobj *bo)
 
 	assert(bo->fetch_objcore->boc->state < BOS_FINISHED);
 	ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED);
-	HSH_Fail(bo->fetch_objcore);
 	if (!(bo->fetch_objcore->flags & OC_F_BUSY))
 		HSH_Kill(bo->fetch_objcore);
 	return (F_STP_DONE);

bin/varnishd/cache/cache_obj.c

@@ -84,6 +84,7 @@
 #include <stdlib.h>
 
 #include "cache_varnishd.h"
+#include "cache_objhead.h"
 #include "cache_obj.h"
 #include "vend.h"
 #include "storage/storage.h"
@@ -257,8 +258,7 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l)
  */
 
 void
-ObjSetState(struct worker *wrk, const struct objcore *oc,
-    enum boc_state_e next)
+ObjSetState(struct worker *wrk, struct objcore *oc, enum boc_state_e next)
 {
 	const struct obj_methods *om;
 
@@ -275,6 +275,13 @@ ObjSetState(struct worker *wrk, const struct objcore *oc,
 			om->objsetstate(wrk, oc, next);
 	}
 
+	if (next == BOS_FAILED) {
+		/* Signal to cache_hash.c that this object is failed. This
+		 * needs to happen before we signal the boc so that code
+		 * will see the OC_F_FAILED flag after ObjWaitState() */
+		HSH_Fail(oc);
+	}
+
 	Lck_Lock(&oc->boc->mtx);
 	oc->boc->state = next;
 	AZ(pthread_cond_broadcast(&oc->boc->cond));

bin/varnishd/cache/cache_varnishd.h

@@ -293,8 +293,7 @@ int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr);
 void ObjExtend(struct worker *, struct objcore *, ssize_t l);
 uint64_t ObjWaitExtend(const struct worker *, const struct objcore *,
     uint64_t l);
-void ObjSetState(struct worker *, const struct objcore *,
-    enum boc_state_e next);
+void ObjSetState(struct worker *, struct objcore *, enum boc_state_e next);
 void ObjWaitState(const struct objcore *, enum boc_state_e want);
 void ObjTrimStore(struct worker *, struct objcore *);
 void ObjTouch(struct worker *, struct objcore *, vtim_real now);