Index: src/Boo.Lang.Compiler/IO/FileInput.cs =================================================================== --- src/Boo.Lang.Compiler/IO/FileInput.cs (revision 2300) +++ src/Boo.Lang.Compiler/IO/FileInput.cs (working copy) @@ -61,7 +61,7 @@ { throw CompilerErrorFactory.FileNotFound(_fname); } - return System.IO.File.OpenText(_fname); + return Kitsu.Boo.Lang.FileEncodingAutoDetection.OpenText(_fname); } } } Index: src/Boo.Lang.Parser/BooParser.cs =================================================================== --- src/Boo.Lang.Parser/BooParser.cs (revision 2300) +++ src/Boo.Lang.Parser/BooParser.cs (working copy) @@ -66,7 +66,7 @@ throw new ArgumentNullException("fname"); } - using (StreamReader reader = File.OpenText(fname)) + using (StreamReader reader = Kitsu.Boo.Lang.FileEncodingAutoDetection.OpenText(fname)) { return ParseReader(tabSize, fname, reader); } Index: src/Boo.Lang/Kitsu/FileEncodingAutoDetection.cs =================================================================== --- src/Boo.Lang/Kitsu/FileEncodingAutoDetection.cs (revision 0) +++ src/Boo.Lang/Kitsu/FileEncodingAutoDetection.cs (revision 0) @@ -0,0 +1,249 @@ +#region license +// Copyright (c) 2006, Snaury (snaury@gmail.com) +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// * Neither the name of the author nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using System; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Collections; + +namespace Kitsu.Boo.Lang +{ + public class FileEncodingAutoDetection + { + private static readonly Regex coding_pep_re = new Regex(@"^(?:#|//).*coding[:=]\s*([-\.\w]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex coding_pep_cp_re = new Regex(@"^(?:cp-?)?(\d+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + public static Encoding DetectEncoding(FileStream fs, IList lines, int maxlines, Encoding defaultEncoding) + { + if(fs == null) + throw new ArgumentNullException("fs"); + if(lines == null) + maxlines = 0; + if(maxlines < 0) + throw new ArgumentOutOfRangeException("maxlines", "maxlines must be greater or equal to zero"); + if(defaultEncoding == null) + defaultEncoding = Encoding.Default; + Encoding ascii = Encoding.ASCII; + long saved_position = fs.Position; + int utf8len = 0; + int numbytes = 0; + int numchars = 0; + int foundlines = 0; + char ch1 = '\0', ch2; + bool have_valid_utf8 = true; + byte[] bytes = new byte[4096]; + StringBuilder sb = null; + char[] chars = null; + if(maxlines > 0) + { + sb = new StringBuilder(); + chars = new char[4096]; // should always be enough + } + while((numbytes = fs.Read(bytes, 0, bytes.Length)) != 0) + { + // scan for needed lines + if(foundlines < maxlines) + { + int linestart = 0, lineend; + numchars = ascii.GetChars(bytes, 0, numbytes, chars, 0); + while(foundlines < maxlines && linestart < numchars) + { + for(lineend = linestart; lineend < numchars; ++lineend) + if(chars[lineend] == '\r' || chars[lineend] == '\n') + break; + if(lineend == numchars) + break; + if(ch1 != '\0' && lineend == 0) + { + ch2 = chars[lineend]; + if((ch1 == '\r' && ch2 == '\n') || (ch1 == '\n' && ch2 == '\r')) + { + linestart = 1; + ch1 = '\0'; + continue; + } + } + if(linestart < lineend) + sb.Append(chars, linestart, lineend - linestart); + lines.Add(sb.ToString()); + sb.Length = 0; + ++foundlines; + if(lineend == (numchars - 1)) + { + linestart = lineend + 1; + ch1 = chars[lineend]; + break; + } + ch1 = chars[lineend]; + ch2 = chars[lineend + 1]; + if((ch1 == '\r' && ch2 == '\n') || (ch1 == '\n' && ch2 == '\r')) + linestart = lineend + 2; + else + linestart = lineend + 1; + ch1 = '\0'; + } + if(foundlines < maxlines && linestart < numchars) + sb.Append(chars, linestart, numchars - linestart); + } + // determine if utf-8 is correct + for(int i = 0; have_valid_utf8 && i < numbytes; ++i) + { + byte b = bytes[i]; + if(b >= 0x80) + { + if(b < 0xc0) + { + if(utf8len > 0) + utf8len--; + else + have_valid_utf8 = false; + } + else if(b >= 0xc2 && b < 0xf5) + { + if(utf8len > 0) + have_valid_utf8 = false; + else if(b < 0xe0) + utf8len = 1; + else if(b < 0xf0) + utf8len = 2; + else + utf8len = 3; + } + else + { + have_valid_utf8 = false; + } + } + else if(utf8len > 0) + have_valid_utf8 = false; + } + if(!have_valid_utf8) + { + if(foundlines < maxlines) + continue; + break; + } + } + fs.Position = saved_position; + if(foundlines < maxlines) + lines.Add(sb.ToString()); + if(have_valid_utf8) + return Encoding.UTF8; + return defaultEncoding; + } + + public static Encoding DetectEncoding(FileStream fs, IList lines, int maxlines) + { + return DetectEncoding(fs, lines, maxlines, null); + } + + public static Encoding DetectEncoding(FileStream fs, Encoding defaultEncoding) + { + return DetectEncoding(fs, null, 0, defaultEncoding); + } + + public static Encoding DetectEncoding(FileStream fs) + { + return DetectEncoding(fs, null, 0, null); + } + + public static StreamReader OpenText(string fname, Encoding defaultEncoding) + { + if(fname == null) + throw new ArgumentNullException("fname"); + FileStream fs = File.OpenRead(fname); + try + { + Encoding enc = Encoding.ASCII; + StreamReader reader = new StreamReader(fs, enc); + reader.Peek(); // force BOM detection + if(reader.CurrentEncoding != enc) + return reader; // BOM detected + reader.DiscardBufferedData(); + reader = null; + + fs.Position = 0; + ArrayList lines = new ArrayList(); + enc = DetectEncoding(fs, lines, 2, defaultEncoding); + foreach(string line in lines) + { + Match m1 = coding_pep_re.Match(line); + if(!m1.Success) + continue; + // support for ascii and mbcs encodings + if(m1.Groups[1].Value == "ascii") + { + enc = Encoding.ASCII; + break; + } + else if(m1.Groups[1].Value == "mbcs") + { + enc = Encoding.Default; + break; + } + Exception innerException = null; + try + { + enc = Encoding.GetEncoding(m1.Groups[1].Value); + break; + } + catch (Exception e) + { + innerException = e; + } + Match m2 = coding_pep_cp_re.Match(m1.Groups[1].Value); + if(m2.Success) + { + try + { + enc = Encoding.GetEncoding(int.Parse(m2.Groups[1].Value)); + break; + } + catch (Exception) + { + } + } + throw innerException; + } + reader = new StreamReader(fs, enc, false); + return reader; + } + catch (Exception) + { + fs.Close(); + throw; + } + } + + public static StreamReader OpenText(string fname) + { + return OpenText(fname, null); + } + } +}