inb & outb woes...

I'm trying to write a short program that will display/modify I/O ports
depending on the number of command line args... this is for a PC104
board I'm tinkering with.  It's not working as expected and I can't
find documentation more in depth than the man page... here's the
code... yeah, it's crude...

#include<stdio.h>
#include<asm/io.h>
#include<unistd.h>
#include<errno.h>
#include<string.h>

int main(int argc, char **argv)
{
    unsigned int addr, val;
    switch (argc) {
        case 2:
            addr = strtol(argv[1], (char**)NULL, 16);
            if (addr < 0x400 && ioperm(addr, 1, 1) < 0) {
                printf("ioperm failed: %s\n", strerror(errno));
                return -1;
            } else if (iopl(3) < 0) {
                printf("iopl failed: %s\n", strerror(errno));
                return -1;
            }
            printf("address 0x%x : value 0x%02x\n", addr, inb(addr));
            break;
        case 3:
            addr = strtol(argv[1], (char**)NULL, 16);
            val = strtol(argv[2], (char**)NULL, 16);
            if (addr < 0x400 && ioperm(addr, 1, 1) < 0) {
                printf("ioperm failed: %s\n", strerror(errno));
                return -1;
            } else if (iopl(3) < 0) {
                printf("iopl failed: %s\n", strerror(errno));
                return -1;
            }
            outb(val&0xff, addr);
            break;
        default:
            printf("invalid args\n");
            return -1;
            break;
    }
    return 0;
}

For what it's worth, I've tried compiling with both -O and -O2, I run
as root, etc.  Any suggestions, help, references to documentation or
anything else would be greatly appreciated.  TIA