UTF8编码规则及用C++语言的识别

UTF-8编码规则

UTF-8是一种变长字节编码方式。对于某一个字符的UTF-8编码，如果只有一个字节则其最高二进制位为0；如果是多字节，其第一个字节从最高位开始，连续的二进制位值为1的个数决定了其编码的位数，其余各字节均以10开头。UTF-8最多可用到6个字节。 

6字节 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 

因此那些基本ASCII字符集中的字符（UNICODE兼容ASCII）只需要一个字节的UTF-8编码（7个二进制位）便可以表示。 

判断一个字符串是否全是UTF-8编码的代码

现在网上有一些判断的算法,但在最后都说判断得不准确,究其原因是代码写错了,哈哈, 参考网址中的程序就是,大家可以看一下.

以下是我写的正确的代码:

#include "stdio.h"


bool isNonAsciiUTF8(const char* str, int &len) {
    if (*str == 0) {
        return false;
    }

    unsigned char chr = *str;

    if(chr>=0x80)
    {
        if(chr>=0xFC&&chr<=0xFD)
            len=6;
        else if(chr>=0xF8)
            len=5;
        else if(chr>=0xF0)
            len=4;
        else if(chr>=0xE0)
            len=3;
        else if(chr>=0xC0)
            len=2;
        else
        {
            return false;
        }
    }


    for(int i=1; i<len; i++) {
        chr = *(str+i);
        if (chr == 0) return false;
        if( (chr & 0xC0) != 0x80) {
            return false;
        }
    }

    return true;
}

bool isTextUTF8(const char* str)
{
    unsigned char chr;

    while(*str) {
        chr = *str;
        if((chr & 0x80) == 0) {
            ++str;
            continue;
        } else {
            int len;
            if (!isNonAsciiUTF8(str, len)) {
                return false;
            } else {
                str += len;
                continue;
            }
        }
    }

    return true;
}


void test(char *p) {
    static int i = 0;
    i++;

    if (isTextUTF8(p)) {
        printf("%d is UTF8 text\n", i);
    } else {
        printf("%d isn't UTF8 text\n", i);
    }


}

int main()
{
    char bytes[7];

    bytes[0] = (unsigned char)0x31;
    bytes[1] = 0;
    test(bytes);

    // 11000000 10110001
    bytes[0] = (unsigned char)0xC0;
    bytes[1] = (unsigned char)0xB1;
    bytes[2] = 0;
    test(bytes);

    // 11100000 10000000 10110001
    bytes[0] = (unsigned char)0xE0;
    bytes[1] = (unsigned char)0x80;
    bytes[2] = (unsigned char)0xB1;
    bytes[3] = 0;
    test(bytes);

    // 11110000 10000000 10000000 10110001
    bytes[0] = (unsigned char)0xF0;
    bytes[1] = (unsigned char)0x80;
    bytes[2] = (unsigned char)0x80;
    bytes[3] = (unsigned char)0xB1;
    bytes[4] = 0;
    test(bytes);

    // 11111000 10000000 10000000 10000000 10110001
    bytes[0] = (unsigned char)0xF8;
    bytes[1] = (unsigned char)0x80;
    bytes[2] = (unsigned char)0x80;
    bytes[3] = (unsigned char)0x80;
    bytes[4] = (unsigned char)0xB1;
    bytes[5] = 0;
    test(bytes);

    // 11111100 10000000 10000000 10000000 10000000 10110001
    bytes[0] = (unsigned char)0xFC;
    bytes[1] = (unsigned char)0x80;
    bytes[2] = (unsigned char)0x80;
    bytes[3] = (unsigned char)0x80;
    bytes[4] = (unsigned char)0x80;
    bytes[5] = (unsigned char)0xB1;
    bytes[6] = 0;
    test(bytes);

    return 0;
}

参考文献:

http://blog.csdn.net/sandyen/article/details/1108168

http://archive.cnblogs.com/a/1914592/

http://www.zxbc.cn/html/20081125/68270.html