Replace internal knowledge in jtag3.c by a public API

In certain situations (CRC failure, device locked), that JTAG3
read functions need to return an indication to the caller that
it is OK to proceed, and allow erasing the device anyway.

Historically, the JTAG3 code passed the respective protocol
errors directly (and unexplained) up to the caller, leaving
the decision to the caller how to handle the situation.

Replace that by a more common return value API. New code should
prefer this API instead of any hardcoded return values.

src/avr.c

@@ -439,16 +439,16 @@ int avr_read(PROGRAMMER * pgm, AVRPART * p, char * memtype,
 	(vmem->tags[i] & TAG_ALLOCATED) != 0)
     {
       rc = pgm->read_byte(pgm, p, mem, i, mem->buf + i);
-      if (rc != 0) {
+      if (rc != LIBAVRDUDE_SUCCESS) {
         avrdude_message(MSG_INFO, "avr_read(): error reading address 0x%04lx\n", i);
-        if (rc == -1) {
+        if (rc == LIBAVRDUDE_GENERAL_FAILURE) {
           avrdude_message(MSG_INFO, "    read operation not supported for memory \"%s\"\n",
                           memtype);
-          return -2;
+          return LIBAVRDUDE_NOTSUPPORTED;
         }
         avrdude_message(MSG_INFO, "    read operation failed for memory \"%s\"\n",
                         memtype);
-        return rc;
+        return LIBAVRDUDE_SOFTFAIL;
       }
     }
     report_progress(i, mem->size, NULL);
@@ -1035,14 +1035,14 @@ int avr_signature(PROGRAMMER * pgm, AVRPART * p)
 
   report_progress (0,1,"Reading");
   rc = avr_read(pgm, p, "signature", 0);
-  if (rc < 0) {
+  if (rc < LIBAVRDUDE_SUCCESS) {
     avrdude_message(MSG_INFO, "%s: error reading signature data for part \"%s\", rc=%d\n",
                     progname, p->desc, rc);
     return rc;
   }
   report_progress (1,1,NULL);
 
-  return 0;
+  return LIBAVRDUDE_SUCCESS;
 }
 
 static uint8_t get_fuse_bitmask(AVRMEM * m) {

src/jtag3.c

@@ -313,6 +313,16 @@ static void jtag3_prmsg(PROGRAMMER * pgm, unsigned char * data, size_t len)
   }
 }
 
+static int jtag3_errcode(int status)
+{
+  if (status >= LIBAVRDUDE_SUCCESS)
+    return status;
+  if (status == RSP3_FAIL_OCD_LOCKED ||
+      status == RSP3_FAIL_CRC_FAILURE)
+    return LIBAVRDUDE_SOFTFAIL;
+  return LIBAVRDUDE_GENERAL_FAILURE;
+}
+
 static void jtag3_prevent(PROGRAMMER * pgm, unsigned char * data, size_t len)
 {
   int i;
@@ -850,7 +860,7 @@ int jtag3_recv(PROGRAMMER * pgm, unsigned char **msg) {
       putc('\n', stderr);
     avrdude_message(MSG_NOTICE2, "%s: %s command: timeout/error communicating with programmer (status %d)\n",
                     progname, descr, status);
-    return -1;
+    return LIBAVRDUDE_GENERAL_FAILURE;
   } else if (verbose >= 3) {
     putc('\n', stderr);
     jtag3_prmsg(pgm, *resp, status);
@@ -871,10 +881,11 @@ int jtag3_recv(PROGRAMMER * pgm, unsigned char **msg) {
     status = (*resp)[3];
     free(*resp);
     resp = 0;
-    return -status;
+
+    return jtag3_errcode(status);
   }
 
-  return status;
+  return LIBAVRDUDE_SUCCESS;
 }
 
 
@@ -987,10 +998,10 @@ static int jtag3_program_enable(PROGRAMMER * pgm)
     free(resp);
     PDATA(pgm)->prog_enabled = 1;
 
-    return 0;
+    return LIBAVRDUDE_SUCCESS;
   }
 
-  return status;
+  return jtag3_errcode(status);
 }
 
 static int jtag3_program_disable(PROGRAMMER * pgm)
@@ -1921,7 +1932,7 @@ static int jtag3_read_byte(PROGRAMMER * pgm, AVRPART * p, AVRMEM * mem,
 
   if (!(pgm->flag & PGM_FL_IS_DW))
     if ((status = jtag3_program_enable(pgm)) < 0)
-      return status;
+      return jtag3_errcode(status);
 
   cmd[0] = SCOPE_AVR;
   cmd[1] = CMD3_READ_MEMORY;
@@ -2007,7 +2018,7 @@ static int jtag3_read_byte(PROGRAMMER * pgm, AVRPART * p, AVRMEM * mem,
 
     if (addr == 0) {
       if ((status = jtag3_command(pgm, cmd, 12, &resp, "read memory")) < 0)
-	return status;
+	return jtag3_errcode(status);
 
       signature_cache[0] = resp[4];
       signature_cache[1] = resp[5];
@@ -2057,7 +2068,7 @@ static int jtag3_read_byte(PROGRAMMER * pgm, AVRPART * p, AVRMEM * mem,
   }
 
   if ((status = jtag3_command(pgm, cmd, 12, &resp, "read memory")) < 0)
-    return status;
+    return jtag3_errcode(status);
 
   if (resp[1] != RSP3_DATA ||
       status < (pagesize? pagesize: 1) + 4) {
@@ -2185,7 +2196,7 @@ static int jtag3_write_byte(PROGRAMMER * pgm, AVRPART * p, AVRMEM * mem,
   cmd[13] = data;
 
   if ((status = jtag3_command(pgm, cmd, 14, &resp, "write memory")) < 0)
-    return status;
+    return jtag3_errcode(status);
 
   free(resp);
 
@@ -2316,7 +2327,7 @@ int jtag3_read_sib(PROGRAMMER * pgm, AVRPART * p, char * sib)
   u32_to_b4(cmd + 8, AVR_SIBLEN);
 
   if ((status = jtag3_command(pgm, cmd, 12, &resp, "read SIB")) < 0)
-	return status;
+	return jtag3_errcode(status);
 
   memcpy(sib, resp+3, AVR_SIBLEN);
   sib[AVR_SIBLEN] = 0; // Zero terminate string

src/jtag3_private.h

@@ -145,6 +145,7 @@
 #  define RSP3_FAIL_UNSUPP_MEMORY       0x34 /* unsupported memory type */
 #  define RSP3_FAIL_WRONG_LENGTH        0x35 /* wrong lenth for mem access */
 #  define RSP3_FAIL_OCD_LOCKED          0x44 /* device is locked */
+#  define RSP3_FAIL_CRC_FAILURE         0x47 /* CRC failure in device */
 #  define RSP3_FAIL_NOT_UNDERSTOOD      0x91
 
 /* ICE events */

src/libavrdude.h

@@ -27,6 +27,16 @@
 #include <stdint.h>
 
 typedef uint32_t pinmask_t;
+/*
+ * Values returned by library functions.
+ * Some library functions also return a count, i.e. a positive
+ * number greater than 0.
+ */
+#define LIBAVRDUDE_SUCCESS 0
+#define LIBAVRDUDE_GENERAL_FAILURE (-1)
+#define LIBAVRDUDE_NOTSUPPORTED (-2) // operation not supported
+#define LIBAVRDUDE_SOFTFAIL (-3) // returned by avr_signature() if caller
+                                 // might proceed with chip erase
 
 /* formerly lists.h */
 

src/main.c

@@ -1116,9 +1116,8 @@ int main(int argc, char * argv [])
     usleep(waittime);
     if (init_ok) {
       rc = avr_signature(pgm, p);
-      if (rc != 0) {
-        // -68 == -(0x44) == -(RSP3_FAIL_OCD_LOCKED)
-        if ((rc == -68) && (p->flags & AVRPART_HAS_UPDI) && (attempt < 1)) {
+      if (rc != LIBAVRDUDE_SUCCESS) {
+        if ((rc == LIBAVRDUDE_SOFTFAIL) && (p->flags & AVRPART_HAS_UPDI) && (attempt < 1)) {
           attempt++;
           if (pgm->read_sib) {
              // Read SIB and compare FamilyID